<commit_before>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <sys/stat.h>
#include <iostream>
#include <string>
using namespace std;

#include "config.h"

#include "mon/MonMap.h"
#include "mds/MDS.h"
#include "osd/OSDMap.h"

#include "msg/SimpleMessenger.h"

#include "common/Timer.h"
#include "common/common_init.h"

#include "mon/MonClient.h"

#include "osdc/Objecter.h"

#include "messages/MOSDGetMap.h"
#include "messages/MClientMount.h"
#include "messages/MClientMountAck.h"

#include "libs3.h"

void usage()
{
  cerr << "usage: c3 -i name [flags] [--mds rank] [--shadow rank]\n";
  cerr << "  -m monitorip:port\n";
  cerr << "        connect to monitor at given address\n";
  cerr << "  --debug_mds n\n";
  cerr << "        debug MDS level (e.g. 10)\n";
  generic_server_usage();
}

class C3 : public Dispatcher
{
  MonMap monmap;
  OSDMap osdmap;
  Messenger *messenger;
  MonClient *mc;
  SimpleMessenger rank;

  bool _dispatch(Message *m);
  bool dispatch_impl(Message *m);

  Objecter *objecter;

  Mutex lock;

 class C_WriteAck : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteAck(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_WriteCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ExecCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_ExecCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ReadCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    bufferlist *bl;
    Cond *pcond;
  public:
    tid_t tid;
    C_ReadCommit(object_t o, loff_t s, size_t *l, bufferlist *b, Cond *cond) : oid(o), start(s), length(l), bl(b),
        pcond(cond) {}
    void finish(int r) {
      *length = r;
      if (pcond)
        pcond->Signal();
    }
  };

public:
  C3() : messenger(NULL), mc(NULL), lock("c3") {}
  ~C3();
  bool init();

  int write(object_t& oid, const char *buf, off_t off, size_t len);
  int exec(object_t& oid, const char *code, off_t data_off, size_t data_len, char *buf, size_t out_len);
  int read(object_t& oid, char *buf, off_t off, size_t len);
};

bool C3::init()
{
  mc = new MonClient(&monmap, NULL);

  // get monmap
  if (!mc->get_monmap())
    return false;

  rank.bind();
  cout << "starting c3." << g_conf.id
       << " at " << rank.get_rank_addr()
       << " fsid " << monmap.get_fsid()
       << std::endl;

  messenger = rank.register_entity(entity_name_t::CLIENT(-1));
  assert_warn(messenger);
  if (!messenger)
    return false;

  mc->set_messenger(messenger);

  rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
  rank.set_policy(entity_name_t::TYPE_MDS, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_OSD, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_CLIENT, SimpleMessenger::Policy::lossless());  // mds does its own timeout/markdown

  rank.start(1);

  mc->mount(g_conf.client_mount_timeout);

  objecter = new Objecter(messenger, &monmap, &osdmap, lock);
  if (!objecter)
    return false;

  lock.Lock();
  mc->link_dispatcher(this);

  objecter->set_client_incarnation(0);
  objecter->init();

  objecter->set_client_incarnation(0);

  lock.Unlock();

  return true;
}

C3::~C3()
{
  if (mc)
    delete mc;
  if (messenger)
    messenger->shutdown();
}

bool C3::dispatch_impl(Message *m)
{
  bool ret;

  // verify protocol version
  if (m->get_orig_source().is_mds() &&
      m->get_header().mds_protocol != CEPH_MDS_PROTOCOL) {
    dout(0) << "mds protocol v " << (int)m->get_header().mds_protocol << " != my " << CEPH_MDS_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  if (m->get_header().mdsc_protocol != CEPH_MDSC_PROTOCOL) {
    dout(0) << "mdsc protocol v " << (int)m->get_header().mdsc_protocol << " != my " << CEPH_MDSC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_mon() &&
      m->get_header().monc_protocol != CEPH_MONC_PROTOCOL) {
    dout(0) << "monc protocol v " << (int)m->get_header().monc_protocol << " != my " << CEPH_MONC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_osd() &&
      m->get_header().osdc_protocol != CEPH_OSDC_PROTOCOL) {
    dout(0) << "osdc protocol v " << (int)m->get_header().osdc_protocol << " != my " << CEPH_OSDC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  lock.Lock();
  ret = _dispatch(m);
  lock.Unlock();

  return ret;
}

bool C3::_dispatch(Message *m)
{
  switch (m->get_type()) {
  // OSD
  case CEPH_MSG_OSD_OPREPLY:
    objecter->handle_osd_op_reply((class MOSDOpReply*)m);
    break;
  case CEPH_MSG_OSD_MAP:
    objecter->handle_osd_map((MOSDMap*)m);
    break;

  default:
    return false;
  }

  return true;
}

int C3::write(object_t& oid, const char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;
  utime_t ut = g_clock.now();

  Mutex lock("C3::write");
  Cond write_wait;
  bl.append(&buf[off], len);

  C_WriteAck *onack = new C_WriteAck(oid, off, &len, &write_wait);
  C_WriteCommit *oncommit = new C_WriteCommit(oid, off, &len, NULL);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to write" << dendl;

  lock.Lock();

  objecter->write(oid, layout,
	      off, len, snapc, bl, ut, 0,
              onack, oncommit);

  dout(0) << "after write call" << dendl;

  write_wait.Wait(lock);

  lock.Unlock();

  return len;
}

int C3::exec(object_t& oid, const char *code, off_t data_off, size_t data_len, char *buf, size_t out_len)
{
  SnapContext snapc;
  bufferlist bl, obl;
  utime_t ut = g_clock.now();

  Mutex lock("C3::exec");
  Cond exec_wait;
  bl.append(code, strlen(code) + 1);

  C_ExecCommit *oncommit = new C_ExecCommit(oid, data_off, &out_len, &exec_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to exec" << dendl;

  lock.Lock();

  objecter->exec(oid, layout,
	      data_off, data_len, CEPH_NOSNAP, bl, 0,
	      &obl, out_len,
              oncommit);

  dout(0) << "after write call" << dendl;

  exec_wait.Wait(lock);

  lock.Unlock();

  if (out_len)
    memcpy(buf, obl.c_str(), out_len);

  return out_len;
}

int C3::read(object_t& oid, char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist *bl = new bufferlist;
  Mutex lock("C3::read");
  Cond read_wait;

  C_ReadCommit *oncommit = new C_ReadCommit(oid, off, &len, bl, &read_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to read" << dendl;

  lock.Lock();

  objecter->read(oid, layout,
	      off, len, CEPH_NOSNAP, bl, 0,
              oncommit);

  dout(0) << "after read call" << dendl;
  read_wait.Wait(lock);

  lock.Unlock();

  if (len)
    memcpy(buf, bl->c_str(), len);

  delete bl;

  return len;
}

static void __c3_init(int argc, const char *argv[])
{
  vector<const char*> args;

  if (argc && argv) {
    argv_to_vec(argc, argv, args);
    env_to_vec(args);
  }
  common_init(args, "ccc", true);

  if (g_conf.clock_tare) g_clock.tare();
}

static Mutex c3_init_mutex("c3_init");
static int c3_initialized = 0;

static C3 *c3p;

#include "c3.h"

extern "C" int c3_initialize(int argc, const char **argv)
{
  int ret = 0;
  c3_init_mutex.Lock();

  if (!c3_initialized) {
    __c3_init(argc, argv);
    c3p = new C3();

    if (!c3p) {
      ret = ENOMEM;
      goto out;
    }
    c3p->init();
  }
  ++c3_initialized;

out:
  c3_init_mutex.Unlock();
  return ret;
}

extern "C" void c3_deinitialize()
{
  c3_init_mutex.Lock();
  --c3_initialized;

  if (!c3_initialized)
    delete c3p;

  c3p = NULL;

  c3_init_mutex.Unlock();
}

extern "C" int c3_write(object_t *oid, const char *buf, off_t off, size_t len)
{
    return c3p->write(*oid, buf, off, len);
}

extern "C" int c3_read(object_t *oid, char *buf, off_t off, size_t len)
{
    return c3p->read(*oid, buf, off, len);
}

extern "C" int c3_exec(object_t *oid, const char *code, off_t off, size_t len, char *buf, size_t out_len)
{
    return c3p->exec(*oid, code, off, len, buf, out_len);
}

int main(int argc, const char **argv)
{
  if (c3_initialize(argc, argv)) {
    cerr << "error initializing" << std::endl;
    exit(1);
  }

  time_t tm;
  char buf[128], buf2[128];

  time(&tm);
  snprintf(buf, 128, "%s", ctime(&tm));

  object_t oid(0x2010, 0);

  c3_write(&oid, buf, 0, strlen(buf) + 1);
  c3_exec(&oid, "code", 0, 128, buf, 128);
  cerr << "exec result=" << buf << std::endl;
  size_t size = c3_read(&oid, buf2, 0, 128);

  cerr << "read result=" << buf2 << "" << std::endl;
  cerr << "size=" << size << std::endl;

  c3_deinitialize();

  return 0;
}

<commit_msg>c3: handle unhandled message<commit_after>// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <sys/stat.h>
#include <iostream>
#include <string>
using namespace std;

#include "config.h"

#include "mon/MonMap.h"
#include "mds/MDS.h"
#include "osd/OSDMap.h"

#include "msg/SimpleMessenger.h"

#include "common/Timer.h"
#include "common/common_init.h"

#include "mon/MonClient.h"

#include "osdc/Objecter.h"

#include "messages/MOSDGetMap.h"
#include "messages/MClientMount.h"
#include "messages/MClientMountAck.h"

#include "libs3.h"

void usage()
{
  cerr << "usage: c3 -i name [flags] [--mds rank] [--shadow rank]\n";
  cerr << "  -m monitorip:port\n";
  cerr << "        connect to monitor at given address\n";
  cerr << "  --debug_mds n\n";
  cerr << "        debug MDS level (e.g. 10)\n";
  generic_server_usage();
}

class C3 : public Dispatcher
{
  MonMap monmap;
  OSDMap osdmap;
  Messenger *messenger;
  MonClient *mc;
  SimpleMessenger rank;

  bool _dispatch(Message *m);
  bool dispatch_impl(Message *m);

  Objecter *objecter;

  Mutex lock;

 class C_WriteAck : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteAck(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_WriteCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_WriteCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ExecCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    Cond *pcond;
  public:
    tid_t tid;
    C_ExecCommit(object_t o, loff_t s, size_t *l, Cond *cond) : oid(o), start(s), length(l), pcond(cond) {}
    void finish(int r) {
      if (pcond) {
        *length = r;
        pcond->Signal();
      }
    }
  };
  class C_ReadCommit : public Context {
    object_t oid;
    loff_t start;
    size_t *length;
    bufferlist *bl;
    Cond *pcond;
  public:
    tid_t tid;
    C_ReadCommit(object_t o, loff_t s, size_t *l, bufferlist *b, Cond *cond) : oid(o), start(s), length(l), bl(b),
        pcond(cond) {}
    void finish(int r) {
      *length = r;
      if (pcond)
        pcond->Signal();
    }
  };

public:
  C3() : messenger(NULL), mc(NULL), lock("c3") {}
  ~C3();
  bool init();

  int write(object_t& oid, const char *buf, off_t off, size_t len);
  int exec(object_t& oid, const char *code, off_t data_off, size_t data_len, char *buf, size_t out_len);
  int read(object_t& oid, char *buf, off_t off, size_t len);
};

bool C3::init()
{
  mc = new MonClient(&monmap, NULL);

  // get monmap
  if (!mc->get_monmap())
    return false;

  rank.bind();
  cout << "starting c3." << g_conf.id
       << " at " << rank.get_rank_addr()
       << " fsid " << monmap.get_fsid()
       << std::endl;

  messenger = rank.register_entity(entity_name_t::CLIENT(-1));
  assert_warn(messenger);
  if (!messenger)
    return false;

  mc->set_messenger(messenger);

  rank.set_policy(entity_name_t::TYPE_MON, SimpleMessenger::Policy::lossy_fail_after(1.0));
  rank.set_policy(entity_name_t::TYPE_MDS, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_OSD, SimpleMessenger::Policy::lossless());
  rank.set_policy(entity_name_t::TYPE_CLIENT, SimpleMessenger::Policy::lossless());  // mds does its own timeout/markdown

  rank.start(1);

  mc->mount(g_conf.client_mount_timeout);

  objecter = new Objecter(messenger, &monmap, &osdmap, lock);
  if (!objecter)
    return false;

  lock.Lock();
  mc->link_dispatcher(this);

  objecter->set_client_incarnation(0);
  objecter->init();

  objecter->set_client_incarnation(0);

  lock.Unlock();

  return true;
}

C3::~C3()
{
  if (mc)
    delete mc;
  if (messenger)
    messenger->shutdown();
}

bool C3::dispatch_impl(Message *m)
{
  bool ret;

  // verify protocol version
  if (m->get_orig_source().is_mds() &&
      m->get_header().mds_protocol != CEPH_MDS_PROTOCOL) {
    dout(0) << "mds protocol v " << (int)m->get_header().mds_protocol << " != my " << CEPH_MDS_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  if (m->get_header().mdsc_protocol != CEPH_MDSC_PROTOCOL) {
    dout(0) << "mdsc protocol v " << (int)m->get_header().mdsc_protocol << " != my " << CEPH_MDSC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_mon() &&
      m->get_header().monc_protocol != CEPH_MONC_PROTOCOL) {
    dout(0) << "monc protocol v " << (int)m->get_header().monc_protocol << " != my " << CEPH_MONC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }
  if (m->get_orig_source().is_osd() &&
      m->get_header().osdc_protocol != CEPH_OSDC_PROTOCOL) {
    dout(0) << "osdc protocol v " << (int)m->get_header().osdc_protocol << " != my " << CEPH_OSDC_PROTOCOL
	    << " from " << m->get_orig_source_inst() << " " << *m << dendl;
    delete m;
    return true;
  }

  lock.Lock();
  ret = _dispatch(m);
  lock.Unlock();

  return ret;
}

bool C3::_dispatch(Message *m)
{
  switch (m->get_type()) {
  // OSD
  case CEPH_MSG_OSD_OPREPLY:
    objecter->handle_osd_op_reply((class MOSDOpReply*)m);
    break;
  case CEPH_MSG_OSD_MAP:
    objecter->handle_osd_map((MOSDMap*)m);
    break;
  case CEPH_MSG_MDS_MAP:
    break;

  default:
    return false;
  }

  return true;
}

int C3::write(object_t& oid, const char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist bl;
  utime_t ut = g_clock.now();

  Mutex lock("C3::write");
  Cond write_wait;
  bl.append(&buf[off], len);

  C_WriteAck *onack = new C_WriteAck(oid, off, &len, &write_wait);
  C_WriteCommit *oncommit = new C_WriteCommit(oid, off, &len, NULL);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to write" << dendl;

  lock.Lock();

  objecter->write(oid, layout,
	      off, len, snapc, bl, ut, 0,
              onack, oncommit);

  dout(0) << "after write call" << dendl;

  write_wait.Wait(lock);

  lock.Unlock();

  return len;
}

int C3::exec(object_t& oid, const char *code, off_t data_off, size_t data_len, char *buf, size_t out_len)
{
  SnapContext snapc;
  bufferlist bl, obl;
  utime_t ut = g_clock.now();

  Mutex lock("C3::exec");
  Cond exec_wait;
  bl.append(code, strlen(code) + 1);

  C_ExecCommit *oncommit = new C_ExecCommit(oid, data_off, &out_len, &exec_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to exec" << dendl;

  lock.Lock();

  objecter->exec(oid, layout,
	      data_off, data_len, CEPH_NOSNAP, bl, 0,
	      &obl, out_len,
              oncommit);

  dout(0) << "after write call" << dendl;

  exec_wait.Wait(lock);

  lock.Unlock();

  if (out_len)
    memcpy(buf, obl.c_str(), out_len);

  return out_len;
}

int C3::read(object_t& oid, char *buf, off_t off, size_t len)
{
  SnapContext snapc;
  bufferlist *bl = new bufferlist;
  Mutex lock("C3::read");
  Cond read_wait;

  C_ReadCommit *oncommit = new C_ReadCommit(oid, off, &len, bl, &read_wait);

  ceph_object_layout layout = objecter->osdmap->make_object_layout(oid, 0);

  dout(0) << "going to read" << dendl;

  lock.Lock();

  objecter->read(oid, layout,
	      off, len, CEPH_NOSNAP, bl, 0,
              oncommit);

  dout(0) << "after read call" << dendl;
  read_wait.Wait(lock);

  lock.Unlock();

  if (len)
    memcpy(buf, bl->c_str(), len);

  delete bl;

  return len;
}

static void __c3_init(int argc, const char *argv[])
{
  vector<const char*> args;

  if (argc && argv) {
    argv_to_vec(argc, argv, args);
    env_to_vec(args);
  }
  common_init(args, "ccc", true);

  if (g_conf.clock_tare) g_clock.tare();
}

static Mutex c3_init_mutex("c3_init");
static int c3_initialized = 0;

static C3 *c3p;

#include "c3.h"

extern "C" int c3_initialize(int argc, const char **argv)
{
  int ret = 0;
  c3_init_mutex.Lock();

  if (!c3_initialized) {
    __c3_init(argc, argv);
    c3p = new C3();

    if (!c3p) {
      ret = ENOMEM;
      goto out;
    }
    c3p->init();
  }
  ++c3_initialized;

out:
  c3_init_mutex.Unlock();
  return ret;
}

extern "C" void c3_deinitialize()
{
  c3_init_mutex.Lock();
  --c3_initialized;

  if (!c3_initialized)
    delete c3p;

  c3p = NULL;

  c3_init_mutex.Unlock();
}

extern "C" int c3_write(object_t *oid, const char *buf, off_t off, size_t len)
{
    return c3p->write(*oid, buf, off, len);
}

extern "C" int c3_read(object_t *oid, char *buf, off_t off, size_t len)
{
    return c3p->read(*oid, buf, off, len);
}

extern "C" int c3_exec(object_t *oid, const char *code, off_t off, size_t len, char *buf, size_t out_len)
{
    return c3p->exec(*oid, code, off, len, buf, out_len);
}

int main(int argc, const char **argv)
{
  if (c3_initialize(argc, argv)) {
    cerr << "error initializing" << std::endl;
    exit(1);
  }

  time_t tm;
  char buf[128], buf2[128];

  time(&tm);
  snprintf(buf, 128, "%s", ctime(&tm));

  object_t oid(0x2010, 0);

  c3_write(&oid, buf, 0, strlen(buf) + 1);
  c3_exec(&oid, "code", 0, 128, buf, 128);
  cerr << "exec result=" << buf << std::endl;
  size_t size = c3_read(&oid, buf2, 0, 128);

  cerr << "read result=" << buf2 << "" << std::endl;
  cerr << "size=" << size << std::endl;

  c3_deinitialize();

  return 0;
}

<|endoftext|>


<commit_before>/**
 * @file fetcher.cpp
 *
 * @date Nov 21, 2012
 * @author partio
 */

#include "fetcher.h"
#include "plugin_factory.h"
#include "logger_factory.h"
#include <fstream>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem/operations.hpp>
#include "timer_factory.h"
#include "util.h"

#define HIMAN_AUXILIARY_INCLUDE

#include "grib.h"
#include "neons.h"
#include "param.h"
#include "cache.h"
#include "querydata.h"

#undef HIMAN_AUXILIARY_INCLUDE

using namespace himan::plugin;
using namespace std;

const unsigned int SLEEPSECONDS = 10;

shared_ptr<cache> itsCache;

fetcher::fetcher()
	: itsDoLevelTransform(true)
{
	itsLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("fetcher"));
}

shared_ptr<himan::info> fetcher::Fetch(shared_ptr<const plugin_configuration> config,
										const forecast_time& requestedTime,
										const level& requestedLevel,
										const params& requestedParams,
										bool readPackedData)
{
	unsigned int waitedSeconds = 0;

	shared_ptr<info> ret;

	do
	{
		for (size_t i = 0; i < requestedParams.size(); i++)
		{
			try
			{
				ret = Fetch(config, requestedTime, requestedLevel, requestedParams[i], readPackedData, false);

				return ret;
			}
			catch (const HPExceptionType& e)
			{
				if (e != kFileDataNotFound)
				{
					throw e;
				}
			}
			catch (const exception& e)
			{
				throw e;
			}

		}
		if (config->FileWaitTimeout() > 0)
		{
			itsLogger->Debug("Sleeping for " + boost::lexical_cast<string> (SLEEPSECONDS) + " seconds (cumulative: " + boost::lexical_cast<string> (waitedSeconds) + ")");

			if (!config->ReadDataFromDatabase())
			{
				itsLogger->Warning("file_wait_timeout specified but file read from Neons is disabled");
			}

			sleep(SLEEPSECONDS);
		}

		waitedSeconds += SLEEPSECONDS;
	}
	while (waitedSeconds < config->FileWaitTimeout() * 60);

	string optsStr = "producer(s): ";

	for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
	{
		optsStr += boost::lexical_cast<string> (config->SourceProducer(prodNum).Id()) + ",";
	}

	optsStr = optsStr.substr(0, optsStr.size()-1);

	optsStr += " origintime: " + requestedTime.OriginDateTime()->String() + ", step: " + boost::lexical_cast<string> (requestedTime.Step());

	optsStr += " param(s): ";

	for (size_t i = 0; i < requestedParams.size(); i++)
	{
		optsStr += requestedParams[i].Name() + ",";
	}

	optsStr = optsStr.substr(0, optsStr.size()-1);

	optsStr += " level: " + string(himan::HPLevelTypeToString.at(requestedLevel.Type())) + " " + boost::lexical_cast<string> (requestedLevel.Value());

	itsLogger->Warning("No valid data found with given search options " + optsStr);

	throw kFileDataNotFound;

}

shared_ptr<himan::info> fetcher::Fetch(shared_ptr<const plugin_configuration> config,
										const forecast_time& requestedTime,
										const level& requestedLevel,
										const param& requestedParam,
										bool readPackedData,
										bool controlWaitTime)
{

	unique_ptr<timer> t = unique_ptr<timer> (timer_factory::Instance()->GetTimer());

	if (config->StatisticsEnabled())
	{
		t->Start();
	}

	vector<shared_ptr<info>> theInfos;
	unsigned int waitedSeconds = 0;

	level newLevel = requestedLevel;

	if (itsDoLevelTransform &&
			(requestedLevel.Type() != kHybrid && requestedLevel.Type() != kPressure))
	{
		newLevel = LevelTransform(config->SourceProducer(), requestedParam, requestedLevel);

		if (newLevel != requestedLevel)
		{
			itsLogger->Trace("Transform level " + string(HPLevelTypeToString.at(requestedLevel.Type()))
						+ "/" + boost::lexical_cast<string> (requestedLevel.Value())
						+ " to " + HPLevelTypeToString.at(newLevel.Type())
						+ "/" + boost::lexical_cast<string> (newLevel.Value())
						+ " for producer " + boost::lexical_cast<string> (config->SourceProducer().Id())
						+ ", parameter " + requestedParam.Name());
		}
	}

	do
	{
		// Loop over all source producers if more than one specified

		for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
		{

			producer sourceProd(config->SourceProducer(prodNum));

			const search_options opts (requestedTime, requestedParam, newLevel, sourceProd, config);

			// itsLogger->Trace("Current producer: " + sourceProd.Name());

			if (config->UseCache())
			{

				// 1. Fetch data from cache

				if (!itsCache)
				{
					itsCache = dynamic_pointer_cast<plugin::cache> (plugin_factory::Instance()->Plugin("cache"));
				}

				theInfos = FromCache(opts);

				if (theInfos.size())
				{
					itsLogger->Trace("Data found from cache");

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheHitCount(1);
					}

					break;
				}
			}

			/*
			 *  2. Fetch data from auxiliary files specified at command line
			 *
			 *  Even if file_wait_timeout is specified, auxiliary files is searched
			 *  only once.
			 */

			if (config->AuxiliaryFiles().size() && waitedSeconds == 0)
			{
				theInfos = FromFile(config->AuxiliaryFiles(), opts, true, readPackedData);

				if (theInfos.size())
				{
					itsLogger->Trace("Data found from auxiliary file(s)");

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheMissCount(1);
					}

					break;
				}
				else
				{
					itsLogger->Trace("Data not found from auxiliary file(s)");
				}
			}

			// 3. Fetch data from Neons

			vector<string> files;

			if (config->ReadDataFromDatabase())
			{
				shared_ptr<neons> n = dynamic_pointer_cast<neons> (plugin_factory::Instance()->Plugin("neons"));

				files = n->Files(opts);

				if (!files.empty())
				{
					theInfos = FromFile(files, opts, true, readPackedData);

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheMissCount(1);
					}

					break;
				}
			}
		}

		if (controlWaitTime && config->FileWaitTimeout() > 0)
		{
			itsLogger->Debug("Sleeping for " + boost::lexical_cast<string> (SLEEPSECONDS) + " seconds (cumulative: " + boost::lexical_cast<string> (waitedSeconds) + ")");

			if (!config->ReadDataFromDatabase())
			{
				itsLogger->Warning("file_wait_timeout specified but file read from Neons is disabled");
			}

			sleep(SLEEPSECONDS);
		}

		waitedSeconds += SLEEPSECONDS;
	}
	while (controlWaitTime && waitedSeconds < config->FileWaitTimeout() * 60);

	if (config->StatisticsEnabled())
	{
		t->Stop();

		config->Statistics()->AddToFetchingTime(t->GetTime());
	}
	/*
	 *  Safeguard; later in the code we do not check whether the data requested
	 *  was actually what was requested.
	 */

	if (theInfos.size() == 0)
	{
		// If this function is called from multi-param Fetch(), do not print
		// any messages yet since we might have another source param coming

		if (controlWaitTime)
		{
			string optsStr = "producer(s): ";

			for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
			{
				optsStr += boost::lexical_cast<string> (config->SourceProducer(prodNum).Id()) + ",";
			}

			optsStr = optsStr.substr(0, optsStr.size()-1);

			optsStr += " origintime: " + requestedTime.OriginDateTime()->String() + ", step: " + boost::lexical_cast<string> (requestedTime.Step());
			optsStr += " param: " + requestedParam.Name();
			optsStr += " level: " + string(himan::HPLevelTypeToString.at(requestedLevel.Type())) + " " + boost::lexical_cast<string> (requestedLevel.Value());

			itsLogger->Warning("No valid data found with given search options " + optsStr);
		}

		throw kFileDataNotFound;
	}

	// assert(theConfiguration->SourceProducer() == theInfos[0]->Producer());

	assert((theInfos[0]->Level()) == newLevel);

	assert((theInfos[0]->Time()) == requestedTime);

	assert((theInfos[0]->Param()) == requestedParam);

	return theInfos[0];

}

vector<shared_ptr<himan::info>> fetcher::FromFile(const vector<string>& files, const search_options& options, bool readContents, bool readPackedData)
{

	vector<shared_ptr<himan::info>> allInfos ;

	for (size_t i = 0; i < files.size(); i++)
	{
		string inputFile = files[i];

		if (!boost::filesystem::exists(inputFile))
		{
			itsLogger->Error("Input file '" + inputFile + "' does not exist");
			continue;
		}

		vector<shared_ptr<himan::info>> curInfos;

		switch (util::FileType(inputFile))
		{

		case kGRIB:
		case kGRIB1:
		case kGRIB2:
		{

			curInfos = FromGrib(inputFile, options, readContents, readPackedData);
			break;

		}

		case kQueryData:
		{
			curInfos = FromQueryData(inputFile, options, readContents);
			break;
		}

		case kNetCDF:
			itsLogger->Error("File is NetCDF");
			break;

		default:
			// Unknown file type, cannot proceed
			throw runtime_error("Input file is neither GRIB, NetCDF nor QueryData");
			break;
		}

		allInfos.insert(allInfos.end(), curInfos.begin(), curInfos.end());

		if (curInfos.size())
		{
			if (options.configuration->UseCache())
			{
				itsCache->Insert(allInfos);
			}

			break; // We found what we were looking for
		}

	}

	return allInfos;
}

vector<shared_ptr<himan::info> > fetcher::FromCache(const search_options& options)
{
	vector<shared_ptr<himan::info>> infos = itsCache->GetInfo(options);

	return infos;
}

vector<shared_ptr<himan::info> > fetcher::FromGrib(const string& inputFile, const search_options& options, bool readContents, bool readPackedData)
{

	shared_ptr<grib> g = dynamic_pointer_cast<grib> (plugin_factory::Instance()->Plugin("grib"));

	vector<shared_ptr<info>> infos = g->FromFile(inputFile, options, readContents, readPackedData);

	return infos;
}

vector<shared_ptr<himan::info>> fetcher::FromQueryData(const string& inputFile, const search_options& options, bool readContents)
{

	shared_ptr<querydata> q = dynamic_pointer_cast<querydata> (plugin_factory::Instance()->Plugin("querydata"));

	shared_ptr<info> i = q->FromFile(inputFile, options, readContents);

	vector<shared_ptr<info>> theInfos;

	theInfos.push_back(i);

	return theInfos;
}

himan::level fetcher::LevelTransform(const producer& sourceProducer, const param& targetParam,	const level& targetLevel) const
{

	level sourceLevel = targetLevel;

	if (sourceProducer.TableVersion() != kHPMissingInt)
	{
		shared_ptr<neons> n = dynamic_pointer_cast <neons> (plugin_factory::Instance()->Plugin("neons"));

		string lvlName = n->NeonsDB().GetGridLevelName(targetParam.Name(), targetLevel.Type(), 204, sourceProducer.TableVersion());

		HPLevelType lvlType = kUnknownLevel;

		double lvlValue = targetLevel.Value();

		if (lvlName == "GROUND")
		{
			lvlType = kGround;
			lvlValue = 0;
		}
		else if (lvlName == "PRESSURE")
		{
			lvlType = kPressure;
		}
		else if (lvlName == "HYBRID")
		{
			lvlType = kHybrid;
		}
		else if (lvlName == "HEIGHT")
		{
			lvlType = kHeight;
		}
		else
		{
			throw runtime_error(ClassName() + ": Unknown level type: " + lvlName);
		}

		sourceLevel = level(lvlType, lvlValue, lvlName);
	}
	else
	{
		sourceLevel = targetLevel;
	}

	return sourceLevel;
}

void fetcher::DoLevelTransform(bool theDoLevelTransform)
{
	itsDoLevelTransform = theDoLevelTransform;
}

bool fetcher::DoLevelTransform() const
{
	return itsDoLevelTransform;
}<commit_msg>Move level transform to inner loop so that all producers are handled correctly<commit_after>/**
 * @file fetcher.cpp
 *
 * @date Nov 21, 2012
 * @author partio
 */

#include "fetcher.h"
#include "plugin_factory.h"
#include "logger_factory.h"
#include <fstream>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem/operations.hpp>
#include "timer_factory.h"
#include "util.h"

#define HIMAN_AUXILIARY_INCLUDE

#include "grib.h"
#include "neons.h"
#include "param.h"
#include "cache.h"
#include "querydata.h"

#undef HIMAN_AUXILIARY_INCLUDE

using namespace himan::plugin;
using namespace std;

const unsigned int SLEEPSECONDS = 10;

shared_ptr<cache> itsCache;

fetcher::fetcher()
	: itsDoLevelTransform(true)
{
	itsLogger = std::unique_ptr<logger> (logger_factory::Instance()->GetLog("fetcher"));
}

shared_ptr<himan::info> fetcher::Fetch(shared_ptr<const plugin_configuration> config,
										const forecast_time& requestedTime,
										const level& requestedLevel,
										const params& requestedParams,
										bool readPackedData)
{
	unsigned int waitedSeconds = 0;

	shared_ptr<info> ret;

	do
	{
		for (size_t i = 0; i < requestedParams.size(); i++)
		{
			try
			{
				ret = Fetch(config, requestedTime, requestedLevel, requestedParams[i], readPackedData, false);

				return ret;
			}
			catch (const HPExceptionType& e)
			{
				if (e != kFileDataNotFound)
				{
					throw;
				}
			}
			catch (const exception& e)
			{
				throw;
			}

		}
		if (config->FileWaitTimeout() > 0)
		{
			itsLogger->Debug("Sleeping for " + boost::lexical_cast<string> (SLEEPSECONDS) + " seconds (cumulative: " + boost::lexical_cast<string> (waitedSeconds) + ")");

			if (!config->ReadDataFromDatabase())
			{
				itsLogger->Warning("file_wait_timeout specified but file read from Neons is disabled");
			}

			sleep(SLEEPSECONDS);
		}

		waitedSeconds += SLEEPSECONDS;
	}
	while (waitedSeconds < config->FileWaitTimeout() * 60);

	string optsStr = "producer(s): ";

	for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
	{
		optsStr += boost::lexical_cast<string> (config->SourceProducer(prodNum).Id()) + ",";
	}

	optsStr = optsStr.substr(0, optsStr.size()-1);

	optsStr += " origintime: " + requestedTime.OriginDateTime()->String() + ", step: " + boost::lexical_cast<string> (requestedTime.Step());

	optsStr += " param(s): ";

	for (size_t i = 0; i < requestedParams.size(); i++)
	{
		optsStr += requestedParams[i].Name() + ",";
	}

	optsStr = optsStr.substr(0, optsStr.size()-1);

	optsStr += " level: " + string(himan::HPLevelTypeToString.at(requestedLevel.Type())) + " " + boost::lexical_cast<string> (requestedLevel.Value());

	itsLogger->Warning("No valid data found with given search options " + optsStr);

	throw kFileDataNotFound;

}

shared_ptr<himan::info> fetcher::Fetch(shared_ptr<const plugin_configuration> config,
										const forecast_time& requestedTime,
										const level& requestedLevel,
										const param& requestedParam,
										bool readPackedData,
										bool controlWaitTime)
{

	unique_ptr<timer> t = unique_ptr<timer> (timer_factory::Instance()->GetTimer());

	if (config->StatisticsEnabled())
	{
		t->Start();
	}

	vector<shared_ptr<info>> theInfos;
	unsigned int waitedSeconds = 0;

	level newLevel = requestedLevel;

	do
	{
		// Loop over all source producers if more than one specified

		for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
		{

			producer sourceProd(config->SourceProducer(prodNum));

			if (itsDoLevelTransform &&
					(requestedLevel.Type() != kHybrid && requestedLevel.Type() != kPressure))
			{
				newLevel = LevelTransform(sourceProd, requestedParam, requestedLevel);

				if (newLevel != requestedLevel)
				{
					itsLogger->Trace("Transform level " + string(HPLevelTypeToString.at(requestedLevel.Type()))
								+ "/" + boost::lexical_cast<string> (requestedLevel.Value())
								+ " to " + HPLevelTypeToString.at(newLevel.Type())
								+ "/" + boost::lexical_cast<string> (newLevel.Value())
								+ " for producer " + boost::lexical_cast<string> (config->SourceProducer().Id())
								+ ", parameter " + requestedParam.Name());
				}
			}

			const search_options opts (requestedTime, requestedParam, newLevel, sourceProd, config);

			// itsLogger->Trace("Current producer: " + sourceProd.Name());

			if (config->UseCache())
			{

				// 1. Fetch data from cache

				if (!itsCache)
				{
					itsCache = dynamic_pointer_cast<plugin::cache> (plugin_factory::Instance()->Plugin("cache"));
				}

				theInfos = FromCache(opts);

				if (theInfos.size())
				{
					itsLogger->Trace("Data found from cache");

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheHitCount(1);
					}

					break;
				}
			}

			/*
			 *  2. Fetch data from auxiliary files specified at command line
			 *
			 *  Even if file_wait_timeout is specified, auxiliary files is searched
			 *  only once.
			 */

			if (config->AuxiliaryFiles().size() && waitedSeconds == 0)
			{
				theInfos = FromFile(config->AuxiliaryFiles(), opts, true, readPackedData);

				if (theInfos.size())
				{
					itsLogger->Trace("Data found from auxiliary file(s)");

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheMissCount(1);
					}

					break;
				}
				else
				{
					itsLogger->Trace("Data not found from auxiliary file(s)");
				}
			}

			// 3. Fetch data from Neons

			vector<string> files;

			if (config->ReadDataFromDatabase())
			{
				shared_ptr<neons> n = dynamic_pointer_cast<neons> (plugin_factory::Instance()->Plugin("neons"));

				files = n->Files(opts);

				if (!files.empty())
				{
					theInfos = FromFile(files, opts, true, readPackedData);

					if (config->StatisticsEnabled())
					{
						config->Statistics()->AddToCacheMissCount(1);
					}

					break;
				}
			}
		}

		if (controlWaitTime && config->FileWaitTimeout() > 0)
		{
			itsLogger->Debug("Sleeping for " + boost::lexical_cast<string> (SLEEPSECONDS) + " seconds (cumulative: " + boost::lexical_cast<string> (waitedSeconds) + ")");

			if (!config->ReadDataFromDatabase())
			{
				itsLogger->Warning("file_wait_timeout specified but file read from Neons is disabled");
			}

			sleep(SLEEPSECONDS);
		}

		waitedSeconds += SLEEPSECONDS;
	}
	while (controlWaitTime && waitedSeconds < config->FileWaitTimeout() * 60);

	if (config->StatisticsEnabled())
	{
		t->Stop();

		config->Statistics()->AddToFetchingTime(t->GetTime());
	}
	/*
	 *  Safeguard; later in the code we do not check whether the data requested
	 *  was actually what was requested.
	 */

	if (theInfos.size() == 0)
	{
		// If this function is called from multi-param Fetch(), do not print
		// any messages yet since we might have another source param coming

		if (controlWaitTime)
		{
			string optsStr = "producer(s): ";

			for (size_t prodNum = 0; prodNum < config->SizeSourceProducers(); prodNum++)
			{
				optsStr += boost::lexical_cast<string> (config->SourceProducer(prodNum).Id()) + ",";
			}

			optsStr = optsStr.substr(0, optsStr.size()-1);

			optsStr += " origintime: " + requestedTime.OriginDateTime()->String() + ", step: " + boost::lexical_cast<string> (requestedTime.Step());
			optsStr += " param: " + requestedParam.Name();
			optsStr += " level: " + string(himan::HPLevelTypeToString.at(requestedLevel.Type())) + " " + boost::lexical_cast<string> (requestedLevel.Value());

			itsLogger->Warning("No valid data found with given search options " + optsStr);
		}

		throw kFileDataNotFound;
	}

	// assert(theConfiguration->SourceProducer() == theInfos[0]->Producer());

	assert((theInfos[0]->Level()) == newLevel);

	assert((theInfos[0]->Time()) == requestedTime);

	assert((theInfos[0]->Param()) == requestedParam);

	return theInfos[0];

}

vector<shared_ptr<himan::info>> fetcher::FromFile(const vector<string>& files, const search_options& options, bool readContents, bool readPackedData)
{

	vector<shared_ptr<himan::info>> allInfos ;

	for (size_t i = 0; i < files.size(); i++)
	{
		string inputFile = files[i];

		if (!boost::filesystem::exists(inputFile))
		{
			itsLogger->Error("Input file '" + inputFile + "' does not exist");
			continue;
		}

		vector<shared_ptr<himan::info>> curInfos;

		switch (util::FileType(inputFile))
		{

		case kGRIB:
		case kGRIB1:
		case kGRIB2:
		{

			curInfos = FromGrib(inputFile, options, readContents, readPackedData);
			break;

		}

		case kQueryData:
		{
			curInfos = FromQueryData(inputFile, options, readContents);
			break;
		}

		case kNetCDF:
			itsLogger->Error("File is NetCDF");
			break;

		default:
			// Unknown file type, cannot proceed
			throw runtime_error("Input file is neither GRIB, NetCDF nor QueryData");
			break;
		}

		allInfos.insert(allInfos.end(), curInfos.begin(), curInfos.end());

		if (curInfos.size())
		{
			if (options.configuration->UseCache())
			{
				itsCache->Insert(allInfos);
			}

			break; // We found what we were looking for
		}

	}

	return allInfos;
}

vector<shared_ptr<himan::info> > fetcher::FromCache(const search_options& options)
{
	vector<shared_ptr<himan::info>> infos = itsCache->GetInfo(options);

	return infos;
}

vector<shared_ptr<himan::info> > fetcher::FromGrib(const string& inputFile, const search_options& options, bool readContents, bool readPackedData)
{

	shared_ptr<grib> g = dynamic_pointer_cast<grib> (plugin_factory::Instance()->Plugin("grib"));

	vector<shared_ptr<info>> infos = g->FromFile(inputFile, options, readContents, readPackedData);

	return infos;
}

vector<shared_ptr<himan::info>> fetcher::FromQueryData(const string& inputFile, const search_options& options, bool readContents)
{

	shared_ptr<querydata> q = dynamic_pointer_cast<querydata> (plugin_factory::Instance()->Plugin("querydata"));

	shared_ptr<info> i = q->FromFile(inputFile, options, readContents);

	vector<shared_ptr<info>> theInfos;

	theInfos.push_back(i);

	return theInfos;
}

himan::level fetcher::LevelTransform(const producer& sourceProducer, const param& targetParam,	const level& targetLevel) const
{

	level sourceLevel = targetLevel;

	if (sourceProducer.TableVersion() != kHPMissingInt)
	{
		shared_ptr<neons> n = dynamic_pointer_cast <neons> (plugin_factory::Instance()->Plugin("neons"));

		string lvlName = n->NeonsDB().GetGridLevelName(targetParam.Name(), targetLevel.Type(), 204, sourceProducer.TableVersion());

		HPLevelType lvlType = kUnknownLevel;

		double lvlValue = targetLevel.Value();

		if (lvlName == "GROUND")
		{
			lvlType = kGround;
			lvlValue = 0;
		}
		else if (lvlName == "PRESSURE")
		{
			lvlType = kPressure;
		}
		else if (lvlName == "HYBRID")
		{
			lvlType = kHybrid;
		}
		else if (lvlName == "HEIGHT")
		{
			lvlType = kHeight;
		}
		else
		{
			throw runtime_error(ClassName() + ": Unknown level type: " + lvlName);
		}

		sourceLevel = level(lvlType, lvlValue, lvlName);
	}
	else
	{
		sourceLevel = targetLevel;
	}

	return sourceLevel;
}

void fetcher::DoLevelTransform(bool theDoLevelTransform)
{
	itsDoLevelTransform = theDoLevelTransform;
}

bool fetcher::DoLevelTransform() const
{
	return itsDoLevelTransform;
}<|endoftext|>


<commit_before>#include "geotiff.h"
#include "cpl_conv.h"  // for CPLMalloc()
#include "file_accessor.h"
#include "gdal_frmts.h"
#include "grid.h"
#include "lambert_conformal_grid.h"
#include "lambert_equal_area_grid.h"
#include "latitude_longitude_grid.h"
#include "logger.h"
#include "plugin_factory.h"
#include "producer.h"
#include "reduced_gaussian_grid.h"
#include "s3.h"
#include "stereographic_grid.h"
#include "timer.h"
#include "transverse_mercator_grid.h"
#include "util.h"
#include <algorithm>
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include <ogr_spatialref.h>

#include "plugin_factory.h"
#include "radon.h"

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#include "gdal_priv.h"

#pragma GCC diagnostic pop

using namespace himan;
using namespace himan::plugin;

#include "radon.h"

geotiff::geotiff()
{
	itsLogger = logger("geotiff");
}

himan::file_information geotiff::ToFile(info<double>& anInfo)
{
	return ToFile<double>(anInfo);
}

template <typename T>
file_information geotiff::ToFile(info<T>& anInfo)
{
	if (anInfo.Grid()->Class() == kIrregularGrid && anInfo.Grid()->Type() != kReducedGaussian)
	{
		itsLogger.Error("Unable to write irregular grid of type " + HPGridTypeToString.at(anInfo.Grid()->Type()) +
		                " to geotiff");
		throw kInvalidWriteOptions;
	}

	itsLogger.Fatal("No support for writing geotiff data");
	himan::Abort();
}

template file_information geotiff::ToFile<double>(info<double>&);
template file_information geotiff::ToFile<float>(info<float>&);

std::unique_ptr<grid> ReadAreaAndGrid(GDALDataset* poDataset)
{
	logger log("geotiff");
	const int ni = poDataset->GetRasterXSize(), nj = poDataset->GetRasterYSize();
	double adfGeoTransform[6];
	if (poDataset->GetGeoTransform(adfGeoTransform) != CE_None)
	{
		log.Error("File does not contain geo transformation coefficients");
		return nullptr;
	}

	const double di = adfGeoTransform[1];
	const double dj = fabs(adfGeoTransform[5]);
	const point fp(adfGeoTransform[0], adfGeoTransform[3]);
	const HPScanningMode sm = (adfGeoTransform[5] < 0) ? kTopLeft : kBottomLeft;
	ASSERT(di > 0);

	std::string proj = poDataset->GetProjectionRef();

	if (proj.empty())
	{
		// Use OCRSpatialReference ?
		return std::unique_ptr<latitude_longitude_grid>(
		    new latitude_longitude_grid(sm, fp, ni, nj, di, dj, earth_shape<double>()));
	}

	const OGRSpatialReference spRef(proj.c_str());
	const std::string projection = spRef.GetAttrValue("PROJECTION");

	if (projection == SRS_PT_LAMBERT_AZIMUTHAL_EQUAL_AREA)
	{
		return std::unique_ptr<lambert_equal_area_grid>(new lambert_equal_area_grid(
		    sm, fp, ni, nj, di, dj, std::unique_ptr<OGRSpatialReference>(spRef.Clone()), true));
	}
	else if (projection == SRS_PT_TRANSVERSE_MERCATOR)
	{
		return std::unique_ptr<transverse_mercator_grid>(new transverse_mercator_grid(
		    sm, fp, ni, nj, di, dj, std::unique_ptr<OGRSpatialReference>(spRef.Clone()), true));
	}
	log.Error("Unsupported projection: " + projection);
	return nullptr;
}

param ReadParam(const std::map<std::string, std::string>& meta, const producer& prod, const param& par)
{
	std::string param_value;

	for (const auto& m : meta)
	{
		if (m.first == "param_name")
		{
			param_value = m.second;
			break;
		}
	}

	if (param_value.empty())
	{
		return par;
	}

	auto r = GET_PLUGIN(radon);

	auto parameter = r->RadonDB().GetParameterFromGeoTIFF(prod.Id(), param_value);

	if (parameter["name"].empty())
	{
		return par;
	}

	return param(parameter["name"]);
}

level ReadLevel(const std::map<std::string, std::string>& meta, const level& lvl)
{
	// Don't know how to deal with this yet
	return lvl;
}

forecast_type ReadForecastType(const std::map<std::string, std::string>& meta, const forecast_type& ftype)
{
	// Don't know how to deal with this yet
	return ftype;
}

void SQLTimeMaskToCTimeMask(std::string& sqlTimeMask)
{
	boost::replace_all(sqlTimeMask, "YYYY", "%Y");
	boost::replace_all(sqlTimeMask, "MM", "%m");
	boost::replace_all(sqlTimeMask, "DD", "%d");
	boost::replace_all(sqlTimeMask, "hh", "%H");
	boost::replace_all(sqlTimeMask, "mm", "%M");
}

forecast_time ReadTime(const std::map<std::string, std::string>& meta, const forecast_time& ftime)
{
	std::string origintime, validtime, mask;
	for (const auto& m : meta)
	{
		if (m.first == "analysis_time")
		{
			origintime = m.second;
		}
		else if (m.first == "valid_time")
		{
			validtime = m.second;
		}
		else if (m.first == "time_mask")
		{
			mask = m.second;
		}
	}

	if (origintime.empty())
	{
		return ftime;
	}

	if (validtime.empty())
	{
		validtime = origintime;
	}

	if (mask.find("YYYY") != std::string::npos)
	{
		SQLTimeMaskToCTimeMask(mask);
	}

	return forecast_time(raw_time(origintime, mask), raw_time(validtime, mask));
}

template <typename T>
GDALDataType TypeToGDALType();

template <>
GDALDataType TypeToGDALType<float>()
{
	return GDT_Float32;
}

template <>
GDALDataType TypeToGDALType<double>()
{
	return GDT_Float64;
}

template <typename T>
T ConvertTo(const std::string& str)
{
	std::istringstream ss(str);
	T num;
	ss >> num;
	return num;
}

std::map<std::string, std::string> ParseMetadata(char** mdata, const producer& prod)
{
	std::map<std::string, std::string> ret;
	std::stringstream ss;
	ss << "SELECT attribute, key, mask FROM geotiff_metadata WHERE producer_id = " << prod.Id();

	auto r = GET_PLUGIN(radon);

	r->RadonDB().Query(ss.str());

	logger log("geotiff");

	while (true)
	{
		const auto row = r->RadonDB().FetchRow();
		if (row.empty())
		{
			break;
		}

		const auto attribute = row[0];
		const auto keyName = row[1];
		const auto keyMask = row[2];
		const char* m = CSLFetchNameValue(mdata, keyName.c_str());

		if (m == nullptr)
		{
			continue;
		}

		const std::string metadata = std::string(m);

		// Try to extract information from free-form text fields
		// attribute: a metadata attribute name that Himan understands, like 'analysis_time'
		// keyName: name of the metadata element in geotiff file
		// keyMask: optional mask for the value of the key, if only specific value needs
		//          to be extraced, regular expressions are used

		if (keyMask.empty())
		{
			ret[attribute] = metadata;
		}
		else
		{
			const boost::regex re(keyMask);
			boost::smatch what;
			if (boost::regex_search(metadata, what, re) == false || what.size() == 0)
			{
				log.Warning("Regex did not match for attribute " + attribute);
				log.Warning("Regex: '" + keyMask + "' Metadata: '" + metadata + "'");
			}

			if (what.size() != 2)
			{
				log.Fatal("Regex matched too many times: " + std::to_string(what.size() - 1));
				himan::Abort();
			}

			log.Debug("Regex match for " + attribute + ": " + what[1]);
			ret[attribute] = what[1];
		}
	}

	return ret;
}

template <typename T>
void ReadData(GDALRasterBand* poBand, matrix<T>& mat, const std::map<std::string, std::string>& meta)
{
	if (meta.find("missing_value") != meta.end())
	{
		mat.MissingValue(ConvertTo<T>(meta.at("missing_value")));
	}

	ASSERT(poBand->GetXSize() == mat.SizeX());
	ASSERT(poBand->GetYSize() == mat.SizeY());

	int nXSize = poBand->GetXSize();
	int nYSize = poBand->GetYSize();

	if (poBand->RasterIO(GF_Read, 0, 0, nXSize, nYSize, mat.ValuesAsPOD(), nXSize, nYSize, TypeToGDALType<T>(), 0, 0) !=
	    CE_None)
	{
		throw std::runtime_error("Read failed");
	}
	// Change missingvalue to our own
	mat.MissingValue(MissingValue<T>());
}

std::vector<std::shared_ptr<info<double>>> geotiff::FromFile(const file_information& theInputFile,
                                                             const search_options& options) const
{
	return FromFile<double>(theInputFile, options);
}

template <typename T>
std::vector<std::shared_ptr<info<T>>> geotiff::FromFile(const file_information& theInputFile,
                                                        const search_options& options) const
{
	std::vector<std::shared_ptr<himan::info<T>>> infos;

	GDALRegister_GTiff();
	// GDALRegister_COG(); // not working, maybe due to using an oldish gdal version

	std::unique_ptr<GDALDataset, std::function<void(GDALDataset*)>> poDataset(
	    reinterpret_cast<GDALDataset*>(GDALOpen(theInputFile.file_location.c_str(), GA_ReadOnly)),
	    [](GDALDataset* dset) { GDALClose(dset); });

	if (poDataset == nullptr)
	{
		itsLogger.Error("Failed to open dataset from " + theInputFile.file_location);
		return infos;
	}

	auto meta = ParseMetadata(poDataset->GetMetadata(), options.prod);  // Get full dataset metadata

	if (meta.size() == 0)
	{
		itsLogger.Error("Failed to parse metadata from " + theInputFile.file_location);
		return infos;
	}

	auto area = ReadAreaAndGrid(poDataset.get());

	if (area == nullptr)
	{
		return infos;
	}

	// "first guess" metadata from file metadata
	auto par = ReadParam(meta, options.prod, options.param);
	auto lvl = ReadLevel(meta, options.level);
	auto ftype = ReadForecastType(meta, options.ftype);
	auto ftime = ReadTime(meta, options.time);

	auto MakeInfoFromGeoTIFFBand = [&](GDALRasterBand* poBand) -> std::shared_ptr<info<T>> {

		// Read possible metadata from band
		auto bmeta = ParseMetadata(poBand->GetMetadata(), options.prod);

		auto bpar = ReadParam(bmeta, options.prod, par);
		auto blvl = ReadLevel(bmeta, lvl);
		auto bftype = ReadForecastType(bmeta, ftype);
		auto bftime = ReadTime(bmeta, ftime);

		if (options.time != bftime)
		{
			itsLogger.Warning("Time does not match: " + options.time.OriginDateTime().String() + " step " +
			                  options.time.Step().String("%02H:%02M:%02S") + " vs " + bftime.OriginDateTime().String() +
			                  " step " + bftime.Step().String("%02H:%02M:%02S"));
		}
		if (options.level != blvl)
		{
			itsLogger.Warning("Level does not match");
		}
		if (options.ftype != bftype)
		{
			itsLogger.Warning("Forecast type does not match");
		}
		if (options.param != bpar)
		{
			itsLogger.Warning("param does not match: " + options.param.Name() + " vs " + bpar.Name());
		}

		auto anInfo = std::make_shared<info<T>>(bftype, bftime, blvl, bpar);
		auto b = std::make_shared<base<T>>();
		b->grid = std::shared_ptr<grid>(area->Clone());

		anInfo->Create(b, true);
		anInfo->Producer(options.prod);

		ReadData<T>(poBand, anInfo->Data(), meta);

		return anInfo;
	};

	if (theInputFile.message_no == boost::none)
	{
		for (int bandNo = 1; bandNo <= poDataset->GetRasterCount(); bandNo++)
		{
			GDALRasterBand* poBand = poDataset->GetRasterBand(bandNo);
			infos.push_back(MakeInfoFromGeoTIFFBand(poBand));
		}
	}
	else
	{
		GDALRasterBand* poBand = poDataset->GetRasterBand(static_cast<int>(theInputFile.message_no.get()));
		infos.push_back(MakeInfoFromGeoTIFFBand(poBand));
	}

	return infos;
}

template std::vector<std::shared_ptr<info<double>>> geotiff::FromFile<double>(const file_information&,
                                                                              const search_options&) const;
template std::vector<std::shared_ptr<info<float>>> geotiff::FromFile<float>(const file_information&,
                                                                            const search_options&) const;
<commit_msg>Log read geotiff file names and band numbers<commit_after>#include "geotiff.h"
#include "cpl_conv.h"  // for CPLMalloc()
#include "file_accessor.h"
#include "gdal_frmts.h"
#include "grid.h"
#include "lambert_conformal_grid.h"
#include "lambert_equal_area_grid.h"
#include "latitude_longitude_grid.h"
#include "logger.h"
#include "plugin_factory.h"
#include "producer.h"
#include "reduced_gaussian_grid.h"
#include "s3.h"
#include "stereographic_grid.h"
#include "timer.h"
#include "transverse_mercator_grid.h"
#include "util.h"
#include <algorithm>
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include <ogr_spatialref.h>

#include "plugin_factory.h"
#include "radon.h"

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#include "gdal_priv.h"

#pragma GCC diagnostic pop

using namespace himan;
using namespace himan::plugin;

#include "radon.h"

geotiff::geotiff()
{
	itsLogger = logger("geotiff");
}

himan::file_information geotiff::ToFile(info<double>& anInfo)
{
	return ToFile<double>(anInfo);
}

template <typename T>
file_information geotiff::ToFile(info<T>& anInfo)
{
	if (anInfo.Grid()->Class() == kIrregularGrid && anInfo.Grid()->Type() != kReducedGaussian)
	{
		itsLogger.Error("Unable to write irregular grid of type " + HPGridTypeToString.at(anInfo.Grid()->Type()) +
		                " to geotiff");
		throw kInvalidWriteOptions;
	}

	itsLogger.Fatal("No support for writing geotiff data");
	himan::Abort();
}

template file_information geotiff::ToFile<double>(info<double>&);
template file_information geotiff::ToFile<float>(info<float>&);

std::unique_ptr<grid> ReadAreaAndGrid(GDALDataset* poDataset)
{
	logger log("geotiff");
	const int ni = poDataset->GetRasterXSize(), nj = poDataset->GetRasterYSize();
	double adfGeoTransform[6];
	if (poDataset->GetGeoTransform(adfGeoTransform) != CE_None)
	{
		log.Error("File does not contain geo transformation coefficients");
		return nullptr;
	}

	const double di = adfGeoTransform[1];
	const double dj = fabs(adfGeoTransform[5]);
	const point fp(adfGeoTransform[0], adfGeoTransform[3]);
	const HPScanningMode sm = (adfGeoTransform[5] < 0) ? kTopLeft : kBottomLeft;
	ASSERT(di > 0);

	std::string proj = poDataset->GetProjectionRef();

	if (proj.empty())
	{
		// Use OCRSpatialReference ?
		return std::unique_ptr<latitude_longitude_grid>(
		    new latitude_longitude_grid(sm, fp, ni, nj, di, dj, earth_shape<double>()));
	}

	const OGRSpatialReference spRef(proj.c_str());
	const std::string projection = spRef.GetAttrValue("PROJECTION");

	if (projection == SRS_PT_LAMBERT_AZIMUTHAL_EQUAL_AREA)
	{
		return std::unique_ptr<lambert_equal_area_grid>(new lambert_equal_area_grid(
		    sm, fp, ni, nj, di, dj, std::unique_ptr<OGRSpatialReference>(spRef.Clone()), true));
	}
	else if (projection == SRS_PT_TRANSVERSE_MERCATOR)
	{
		return std::unique_ptr<transverse_mercator_grid>(new transverse_mercator_grid(
		    sm, fp, ni, nj, di, dj, std::unique_ptr<OGRSpatialReference>(spRef.Clone()), true));
	}
	log.Error("Unsupported projection: " + projection);
	return nullptr;
}

param ReadParam(const std::map<std::string, std::string>& meta, const producer& prod, const param& par)
{
	std::string param_value;

	for (const auto& m : meta)
	{
		if (m.first == "param_name")
		{
			param_value = m.second;
			break;
		}
	}

	if (param_value.empty())
	{
		return par;
	}

	auto r = GET_PLUGIN(radon);

	auto parameter = r->RadonDB().GetParameterFromGeoTIFF(prod.Id(), param_value);

	if (parameter["name"].empty())
	{
		return par;
	}

	return param(parameter["name"]);
}

level ReadLevel(const std::map<std::string, std::string>& meta, const level& lvl)
{
	// Don't know how to deal with this yet
	return lvl;
}

forecast_type ReadForecastType(const std::map<std::string, std::string>& meta, const forecast_type& ftype)
{
	// Don't know how to deal with this yet
	return ftype;
}

void SQLTimeMaskToCTimeMask(std::string& sqlTimeMask)
{
	boost::replace_all(sqlTimeMask, "YYYY", "%Y");
	boost::replace_all(sqlTimeMask, "MM", "%m");
	boost::replace_all(sqlTimeMask, "DD", "%d");
	boost::replace_all(sqlTimeMask, "hh", "%H");
	boost::replace_all(sqlTimeMask, "mm", "%M");
}

forecast_time ReadTime(const std::map<std::string, std::string>& meta, const forecast_time& ftime)
{
	std::string origintime, validtime, mask;
	for (const auto& m : meta)
	{
		if (m.first == "analysis_time")
		{
			origintime = m.second;
		}
		else if (m.first == "valid_time")
		{
			validtime = m.second;
		}
		else if (m.first == "time_mask")
		{
			mask = m.second;
		}
	}

	if (origintime.empty())
	{
		return ftime;
	}

	if (validtime.empty())
	{
		validtime = origintime;
	}

	if (mask.find("YYYY") != std::string::npos)
	{
		SQLTimeMaskToCTimeMask(mask);
	}

	return forecast_time(raw_time(origintime, mask), raw_time(validtime, mask));
}

template <typename T>
GDALDataType TypeToGDALType();

template <>
GDALDataType TypeToGDALType<float>()
{
	return GDT_Float32;
}

template <>
GDALDataType TypeToGDALType<double>()
{
	return GDT_Float64;
}

template <typename T>
T ConvertTo(const std::string& str)
{
	std::istringstream ss(str);
	T num;
	ss >> num;
	return num;
}

std::map<std::string, std::string> ParseMetadata(char** mdata, const producer& prod)
{
	std::map<std::string, std::string> ret;
	std::stringstream ss;
	ss << "SELECT attribute, key, mask FROM geotiff_metadata WHERE producer_id = " << prod.Id();

	auto r = GET_PLUGIN(radon);

	r->RadonDB().Query(ss.str());

	logger log("geotiff");

	while (true)
	{
		const auto row = r->RadonDB().FetchRow();
		if (row.empty())
		{
			break;
		}

		const auto attribute = row[0];
		const auto keyName = row[1];
		const auto keyMask = row[2];
		const char* m = CSLFetchNameValue(mdata, keyName.c_str());

		if (m == nullptr)
		{
			continue;
		}

		const std::string metadata = std::string(m);

		// Try to extract information from free-form text fields
		// attribute: a metadata attribute name that Himan understands, like 'analysis_time'
		// keyName: name of the metadata element in geotiff file
		// keyMask: optional mask for the value of the key, if only specific value needs
		//          to be extraced, regular expressions are used

		if (keyMask.empty())
		{
			ret[attribute] = metadata;
		}
		else
		{
			const boost::regex re(keyMask);
			boost::smatch what;
			if (boost::regex_search(metadata, what, re) == false || what.size() == 0)
			{
				log.Warning("Regex did not match for attribute " + attribute);
				log.Warning("Regex: '" + keyMask + "' Metadata: '" + metadata + "'");
			}

			if (what.size() != 2)
			{
				log.Fatal("Regex matched too many times: " + std::to_string(what.size() - 1));
				himan::Abort();
			}

			log.Debug("Regex match for " + attribute + ": " + what[1]);
			ret[attribute] = what[1];
		}
	}

	return ret;
}

template <typename T>
void ReadData(GDALRasterBand* poBand, matrix<T>& mat, const std::map<std::string, std::string>& meta)
{
	if (meta.find("missing_value") != meta.end())
	{
		mat.MissingValue(ConvertTo<T>(meta.at("missing_value")));
	}

	ASSERT(poBand->GetXSize() == mat.SizeX());
	ASSERT(poBand->GetYSize() == mat.SizeY());

	int nXSize = poBand->GetXSize();
	int nYSize = poBand->GetYSize();

	if (poBand->RasterIO(GF_Read, 0, 0, nXSize, nYSize, mat.ValuesAsPOD(), nXSize, nYSize, TypeToGDALType<T>(), 0, 0) !=
	    CE_None)
	{
		throw std::runtime_error("Read failed");
	}
	// Change missingvalue to our own
	mat.MissingValue(MissingValue<T>());
}

std::vector<std::shared_ptr<info<double>>> geotiff::FromFile(const file_information& theInputFile,
                                                             const search_options& options) const
{
	return FromFile<double>(theInputFile, options);
}

template <typename T>
std::vector<std::shared_ptr<info<T>>> geotiff::FromFile(const file_information& theInputFile,
                                                        const search_options& options) const
{
	std::vector<std::shared_ptr<himan::info<T>>> infos;

	GDALRegister_GTiff();
	// GDALRegister_COG(); // not working, maybe due to using an oldish gdal version

	std::unique_ptr<GDALDataset, std::function<void(GDALDataset*)>> poDataset(
	    reinterpret_cast<GDALDataset*>(GDALOpen(theInputFile.file_location.c_str(), GA_ReadOnly)),
	    [](GDALDataset* dset) { GDALClose(dset); });

	if (poDataset == nullptr)
	{
		itsLogger.Error("Failed to open dataset from " + theInputFile.file_location);
		return infos;
	}

	auto meta = ParseMetadata(poDataset->GetMetadata(), options.prod);  // Get full dataset metadata

	if (meta.size() == 0)
	{
		itsLogger.Error("Failed to parse metadata from " + theInputFile.file_location);
		return infos;
	}

	auto area = ReadAreaAndGrid(poDataset.get());

	if (area == nullptr)
	{
		return infos;
	}

	// "first guess" metadata from file metadata
	auto par = ReadParam(meta, options.prod, options.param);
	auto lvl = ReadLevel(meta, options.level);
	auto ftype = ReadForecastType(meta, options.ftype);
	auto ftime = ReadTime(meta, options.time);

	auto MakeInfoFromGeoTIFFBand = [&](GDALRasterBand* poBand) -> std::shared_ptr<info<T>> {

		// Read possible metadata from band
		auto bmeta = ParseMetadata(poBand->GetMetadata(), options.prod);

		auto bpar = ReadParam(bmeta, options.prod, par);
		auto blvl = ReadLevel(bmeta, lvl);
		auto bftype = ReadForecastType(bmeta, ftype);
		auto bftime = ReadTime(bmeta, ftime);

		if (options.time != bftime)
		{
			itsLogger.Warning("Time does not match: " + options.time.OriginDateTime().String() + " step " +
			                  options.time.Step().String("%02H:%02M:%02S") + " vs " + bftime.OriginDateTime().String() +
			                  " step " + bftime.Step().String("%02H:%02M:%02S"));
		}
		if (options.level != blvl)
		{
			itsLogger.Warning("Level does not match");
		}
		if (options.ftype != bftype)
		{
			itsLogger.Warning("Forecast type does not match");
		}
		if (options.param != bpar)
		{
			itsLogger.Warning("param does not match: " + options.param.Name() + " vs " + bpar.Name());
		}

		auto anInfo = std::make_shared<info<T>>(bftype, bftime, blvl, bpar);
		auto b = std::make_shared<base<T>>();
		b->grid = std::shared_ptr<grid>(area->Clone());

		anInfo->Create(b, true);
		anInfo->Producer(options.prod);

		ReadData<T>(poBand, anInfo->Data(), meta);

		return anInfo;
	};

	if (theInputFile.message_no == boost::none)
	{
		for (int bandNo = 1; bandNo <= poDataset->GetRasterCount(); bandNo++)
		{
			itsLogger.Info("Read from file '" + theInputFile.file_location + "- band# " + std::to_string(bandNo));
			GDALRasterBand* poBand = poDataset->GetRasterBand(bandNo);
			infos.push_back(MakeInfoFromGeoTIFFBand(poBand));
		}
	}
	else
	{
		itsLogger.Info("Read from file '" + theInputFile.file_location + "' band# " +
		               std::to_string(theInputFile.message_no.get()));
		GDALRasterBand* poBand = poDataset->GetRasterBand(static_cast<int>(theInputFile.message_no.get()));
		infos.push_back(MakeInfoFromGeoTIFFBand(poBand));
	}

	return infos;
}

template std::vector<std::shared_ptr<info<double>>> geotiff::FromFile<double>(const file_information&,
                                                                              const search_options&) const;
template std::vector<std::shared_ptr<info<float>>> geotiff::FromFile<float>(const file_information&,
                                                                            const search_options&) const;
<|endoftext|>


<commit_before>
#ifdef _WIN32
#include <Windows.h>
#else
#include <pthread.h>
#endif

#include <nstd/Signal.h>
#include <nstd/Debug.h>

Signal::Signal(bool set)
{
#ifdef _WIN32
  VERIFY(handle = CreateEvent(NULL, TRUE, set ? TRUE : FALSE, NULL));
#else
  ASSERT(sizeof(cdata) >= sizeof(pthread_cond_t));
  ASSERT(sizeof(mdata) >= sizeof(pthread_mutex_t));
  pthread_cond_init((pthread_cond_t*)cdata, 0);
  pthread_mutex_init((pthread_mutex_t*)mdata, 0);
  signaled = set;
#endif
}

Signal::~Signal()
{
#ifdef _WIN32
  VERIFY(CloseHandle(handle));
#else
  VERIFY(pthread_cond_destroy((pthread_cond_t*)cdata) == 0);
  VERIFY(pthread_mutex_destroy((pthread_mutex_t*)mdata) == 0);
#endif
}

void Signal::set()
{
#ifdef _WIN32
  VERIFY(SetEvent(handle));
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  signaled = true;
  VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
  VERIFY(pthread_cond_signal((pthread_cond_t*)cdata) == 0);
#endif
}

void Signal::reset()
{
#ifdef _WIN32
  VERIFY(ResetEvent(handle));
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  signaled = false;
  VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
#endif
}

bool Signal::wait()
{
#ifdef _WIN32
  return WaitForSingleObject(handle, INFINITE) == WAIT_OBJECT_0;
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  for(;;)
  {
    if(signaled)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return true;
    }
    VERIFY(pthread_cond_wait((pthread_cond_t*)cdata, (pthread_mutex_t*)mdata) == 0);
  }
#endif
}

bool Signal::wait(int64 timeout)
{
#ifdef _WIN32
  return WaitForSingleObject(handle, (DWORD)timeout) == WAIT_OBJECT_0;
#else
  struct timespec ts;
  clock_gettime(CLOCK_REALTIME, &ts);
  ts.tv_nsec += (timeout % 1000) * 1000000;
  ts.tv_sec += timeout / 1000 + ts.tv_nsec / 1000000000;
  ts.tv_nsec %= 1000000000;
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  for(;;)
  {
    if(signaled)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return true;
    }
    if(pthread_cond_timedwait((pthread_cond_t*)cdata, (pthread_mutex_t*)mdata, &ts) != 0)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return false;
    }
  }
#endif
}
<commit_msg>Signal: Fixed thread waking on Linux<commit_after>
#ifdef _WIN32
#include <Windows.h>
#else
#include <pthread.h>
#endif

#include <nstd/Signal.h>
#include <nstd/Debug.h>

Signal::Signal(bool set)
{
#ifdef _WIN32
  VERIFY(handle = CreateEvent(NULL, TRUE, set ? TRUE : FALSE, NULL));
#else
  ASSERT(sizeof(cdata) >= sizeof(pthread_cond_t));
  ASSERT(sizeof(mdata) >= sizeof(pthread_mutex_t));
  pthread_cond_init((pthread_cond_t*)cdata, 0);
  pthread_mutex_init((pthread_mutex_t*)mdata, 0);
  signaled = set;
#endif
}

Signal::~Signal()
{
#ifdef _WIN32
  VERIFY(CloseHandle(handle));
#else
  VERIFY(pthread_cond_destroy((pthread_cond_t*)cdata) == 0);
  VERIFY(pthread_mutex_destroy((pthread_mutex_t*)mdata) == 0);
#endif
}

void Signal::set()
{
#ifdef _WIN32
  VERIFY(SetEvent(handle));
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  signaled = true;
  VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
  VERIFY(pthread_cond_broadcast((pthread_cond_t*)cdata) == 0);
#endif
}

void Signal::reset()
{
#ifdef _WIN32
  VERIFY(ResetEvent(handle));
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  signaled = false;
  VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
#endif
}

bool Signal::wait()
{
#ifdef _WIN32
  return WaitForSingleObject(handle, INFINITE) == WAIT_OBJECT_0;
#else
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  for(;;)
  {
    if(signaled)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return true;
    }
    VERIFY(pthread_cond_wait((pthread_cond_t*)cdata, (pthread_mutex_t*)mdata) == 0);
  }
#endif
}

bool Signal::wait(int64 timeout)
{
#ifdef _WIN32
  return WaitForSingleObject(handle, (DWORD)timeout) == WAIT_OBJECT_0;
#else
  struct timespec ts;
  clock_gettime(CLOCK_REALTIME, &ts);
  ts.tv_nsec += (timeout % 1000) * 1000000;
  ts.tv_sec += timeout / 1000 + ts.tv_nsec / 1000000000;
  ts.tv_nsec %= 1000000000;
  VERIFY(pthread_mutex_lock((pthread_mutex_t*)mdata) == 0);
  for(;;)
  {
    if(signaled)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return true;
    }
    if(pthread_cond_timedwait((pthread_cond_t*)cdata, (pthread_mutex_t*)mdata, &ts) != 0)
    {
      VERIFY(pthread_mutex_unlock((pthread_mutex_t*)mdata) == 0);
      return false;
    }
  }
#endif
}
<|endoftext|>


<commit_before>/*
 * This file is part of libArcus
 *
 * Copyright (C) 2015 Ultimaker b.v. <a.hiemstra@ultimaker.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "Socket.h"
#include "Socket_p.h"

#include <algorithm>
#include <typeinfo>

using namespace Arcus;

Socket::Socket() : d(new SocketPrivate)
{
}

Socket::~Socket()
{
    if(d->thread)
    {
        if(d->state != SocketState::Closed || d->state != SocketState::Error)
        {
            d->nextState = SocketState::Closing;
            if(d->thread)
            {
                d->thread->join();
                d->thread = nullptr;
            }
        }
        delete d->thread;
    }
}

SocketState::State Socket::state() const
{
    return d->state;
}

std::string Socket::errorString() const
{
    return d->errorString;
}

void Socket::clearError()
{
    d->errorString.clear();
}

void Socket::registerMessageType(int type, const google::protobuf::Message* messageType)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    if(type <= 0)
        throw new std::bad_typeid();

    d->messageTypes[type] = messageType;
    d->messageTypeMapping[messageType->GetDescriptor()] = type;
}

void Socket::addListener(SocketListener* listener)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    listener->setSocket(this);
    d->listeners.push_back(listener);
}

void Socket::removeListener(SocketListener* listener)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    auto itr = std::find(d->listeners.begin(), d->listeners.end(), listener);
    d->listeners.erase(itr);
}

void Socket::connect(const std::string& address, int port)
{
    d->address = address;
    d->port = port;
    d->nextState = SocketState::Connecting;
    d->thread = new std::thread([&]() { d->run(); });
}

void Socket::reset()
{
    if (d->state != SocketState::Closed &&
        d->state != SocketState::Error)
        return;

    if(d->thread)
    {
            d->thread->join();
            d->thread = nullptr;
    }

    d->state = SocketState::Initial;
    d->nextState = SocketState::Initial;
    d->errorString = "";
}

void Socket::listen(const std::string& address, int port)
{
    d->address = address;
    d->port = port;
    d->nextState = SocketState::Opening;
    d->thread = new std::thread([&]() { d->run(); });
}

void Socket::close()
{
    d->nextState = SocketState::Closing;
    if(d->thread)
    {
        d->thread->join();
        d->thread = nullptr;
    }
}

void Socket::sendMessage(MessagePtr message)
{
    std::lock_guard<std::mutex> lock(d->sendQueueMutex);
    d->sendQueue.push_back(message);
}

MessagePtr Socket::takeNextMessage()
{
    std::lock_guard<std::mutex> lock(d->receiveQueueMutex);
    if(d->receiveQueue.size() > 0)
    {
        MessagePtr next = d->receiveQueue.front();
        d->receiveQueue.pop_front();
        return next;
    }
    else
    {
        return nullptr;
    }
}
<commit_msg>Socket: Only connect if there's no signs of connection<commit_after>/*
 * This file is part of libArcus
 *
 * Copyright (C) 2015 Ultimaker b.v. <a.hiemstra@ultimaker.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "Socket.h"
#include "Socket_p.h"

#include <algorithm>
#include <typeinfo>

using namespace Arcus;

Socket::Socket() : d(new SocketPrivate)
{
}

Socket::~Socket()
{
    if(d->thread)
    {
        if(d->state != SocketState::Closed || d->state != SocketState::Error)
        {
            d->nextState = SocketState::Closing;
            if(d->thread)
            {
                d->thread->join();
                d->thread = nullptr;
            }
        }
        delete d->thread;
    }
}

SocketState::State Socket::state() const
{
    return d->state;
}

std::string Socket::errorString() const
{
    return d->errorString;
}

void Socket::clearError()
{
    d->errorString.clear();
}

void Socket::registerMessageType(int type, const google::protobuf::Message* messageType)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    if(type <= 0)
        throw new std::bad_typeid();

    d->messageTypes[type] = messageType;
    d->messageTypeMapping[messageType->GetDescriptor()] = type;
}

void Socket::addListener(SocketListener* listener)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    listener->setSocket(this);
    d->listeners.push_back(listener);
}

void Socket::removeListener(SocketListener* listener)
{
    if(d->state != SocketState::Initial)
    {
        return;
    }

    auto itr = std::find(d->listeners.begin(), d->listeners.end(), listener);
    d->listeners.erase(itr);
}

void Socket::connect(const std::string& address, int port)
{
    if(d->state != SocketState::Initial || d->thread != nullptr)
    {
        return;
    }

    d->address = address;
    d->port = port;
    d->nextState = SocketState::Connecting;
    d->thread = new std::thread([&]() { d->run(); });
}

void Socket::reset()
{
    if (d->state != SocketState::Closed &&
        d->state != SocketState::Error)
        return;

    if(d->thread)
    {
        d->thread->join();
        d->thread = nullptr;
    }

    d->state = SocketState::Initial;
    d->nextState = SocketState::Initial;
    d->errorString = "";
}

void Socket::listen(const std::string& address, int port)
{
    d->address = address;
    d->port = port;
    d->nextState = SocketState::Opening;
    d->thread = new std::thread([&]() { d->run(); });
}

void Socket::close()
{
    d->nextState = SocketState::Closing;
    if(d->thread)
    {
        d->thread->join();
        d->thread = nullptr;
    }
}

void Socket::sendMessage(MessagePtr message)
{
    std::lock_guard<std::mutex> lock(d->sendQueueMutex);
    d->sendQueue.push_back(message);
}

MessagePtr Socket::takeNextMessage()
{
    std::lock_guard<std::mutex> lock(d->receiveQueueMutex);
    if(d->receiveQueue.size() > 0)
    {
        MessagePtr next = d->receiveQueue.front();
        d->receiveQueue.pop_front();
        return next;
    }
    else
    {
        return nullptr;
    }
}
<|endoftext|>


<commit_before>
/*
 *  Stream.cpp
 *  sfeMovie project
 *
 *  Copyright (C) 2010-2015 Lucas Soltic
 *  lucas.soltic@orange.fr
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 */

extern "C"
{
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
}

#include "Stream.hpp"
#include "Utilities.hpp"
#include "TimerPriorities.hpp"
#include <cassert>
#include <iostream>
#include <stdexcept>

namespace sfe
{
    std::string Stream::AVStreamDescription(AVStream* stream)
    {
        return std::string("'" + std::string(av_get_media_type_string(stream->codec->codec_type))
                           + "/" + avcodec_get_name(stream->codec->codec_id) + "' stream @ " + s(stream));
    }

    Stream::Stream(AVFormatContext*& formatCtx, AVStream*& stream, DataSource& dataSource, std::shared_ptr<Timer> timer) :
    m_formatCtx(formatCtx),
    m_stream(stream),
    m_dataSource(dataSource),
    m_timer(timer),
    m_codec(nullptr),
    m_streamID(-1),
    m_packetList(),
    m_status(Stopped),
    m_readerMutex()
    {
        CHECK(stream, "Stream::Stream() - invalid stream argument");
        CHECK(timer, "Inconcistency error: null timer");
        int err = 0;

        m_stream = stream;
        m_streamID = stream->index;
        CHECK(m_stream, "Inconcistency error: null stream")
        CHECK(m_streamID >= 0, "Inconcistency error: invalid stream id");

        // Get the decoder
        m_codec = avcodec_find_decoder(m_stream->codec->codec_id);
        CHECK(m_codec, "Stream() - no decoder for " + std::string(avcodec_get_name(m_stream->codec->codec_id)) + " codec");

        // Load the codec
        err = avcodec_open2(m_stream->codec, m_codec, nullptr);
        CHECK0(err, "Stream() - unable to load decoder for codec " + std::string(avcodec_get_name(m_stream->codec->codec_id)));

        AVDictionaryEntry* entry = av_dict_get(m_stream->metadata, "language", nullptr, 0);
        if (entry)
        {
            m_language = entry->value;
        }
    }

    Stream::~Stream()
    {
        disconnect();
        flushBuffers();

        if (m_formatCtx && m_stream && m_stream->codec)
        {
            avcodec_close(m_stream->codec);
        }
    }

    void Stream::connect()
    {
        if (isPassive())
            m_timer->addObserver(*this, PassiveStreamTimerPriority);
        else
            m_timer->addObserver(*this, ActiveStreamTimerPriority);
    }

    void Stream::disconnect()
    {
        m_timer->removeObserver(*this);
    }

    void Stream::pushEncodedData(AVPacket* packet)
    {
        CHECK(packet, "invalid argument");
        sf::Lock l(m_readerMutex);
        m_packetList.push_back(packet);
    }

    void Stream::prependEncodedData(AVPacket* packet)
    {
        CHECK(packet, "invalid argument");
        sf::Lock l(m_readerMutex);
        m_packetList.push_front(packet);
    }

    AVPacket* Stream::popEncodedData()
    {
        AVPacket* result = nullptr;
        sf::Lock l(m_readerMutex);

        if (m_packetList.empty() && !isPassive())
        {
            m_dataSource.requestMoreData(*this);
        }

        if (!m_packetList.empty())
        {
            result = m_packetList.front();
            m_packetList.pop_front();
        }
        else
        {
            if (m_stream->codec->codec->capabilities & CODEC_CAP_DELAY)
            {
                AVPacket* flushPacket = (AVPacket*)av_malloc(sizeof(*flushPacket));
                av_init_packet(flushPacket);
                flushPacket->data = nullptr;
                flushPacket->size = 0;
                result = flushPacket;

                sfeLogDebug("Sending flush packet: " + mediaTypeToString(getStreamKind()));
            }
        }

        return result;
    }

    void Stream::flushBuffers()
    {
        sf::Lock l(m_readerMutex);
        if (getStatus() == Playing)
        {
            sfeLogWarning("packets flushed while the stream is still playing");
        }

        if (m_formatCtx && m_stream)
            avcodec_flush_buffers(m_stream->codec);

        AVPacket* pkt = nullptr;

        while (!m_packetList.empty())
        {
            pkt = m_packetList.front();
            m_packetList.pop_front();

            av_free_packet(pkt);
            av_free(pkt);
        }
    }

    bool Stream::needsMoreData() const
    {
        return m_packetList.size() < 10;
    }

    MediaType Stream::getStreamKind() const
    {
        return Unknown;
    }

    Status Stream::getStatus() const
    {
        return m_status;
    }

    std::string Stream::getLanguage() const
    {
        return m_language;
    }

    sf::Time Stream::computeEncodedPosition()
    {
        if (m_packetList.empty())
        {
            m_dataSource.requestMoreData(*this);
            return sf::Time::Zero;
        }
        else
        {
            sf::Lock l(m_readerMutex);
            AVPacket* packet = m_packetList.front();
            CHECK(packet, "internal inconcistency");

            int64_t timestamp = -424242;

            if (packet->dts != AV_NOPTS_VALUE)
            {
                timestamp = packet->dts;
            }
            else if (packet->pts != AV_NOPTS_VALUE)
            {
                int64_t startTime = m_stream->start_time != AV_NOPTS_VALUE ? m_stream->start_time : 0;
                timestamp = packet->pts - startTime;
            }

            AVRational seconds = av_mul_q(av_make_q(timestamp, 1), m_stream->time_base);
            return sf::milliseconds(1000 * av_q2d(seconds));
        }
    }

    sf::Time Stream::packetDuration(const AVPacket* packet) const
    {
        CHECK(packet, "inconcistency error: null packet");
        CHECK(packet->stream_index == m_streamID, "Asking for duration of a packet for a different stream!");

        if (packet->duration != 0)
        {
            AVRational seconds = av_mul_q(av_make_q(packet->duration, 1), m_stream->time_base);
            return sf::seconds(av_q2d(seconds));
        }
        else
        {
            return sf::seconds(1. / av_q2d(av_guess_frame_rate(m_formatCtx, m_stream, nullptr)));
        }
    }

    std::string Stream::description() const
    {
        return AVStreamDescription(m_stream);
    }

    bool Stream::canUsePacket(AVPacket* packet) const
    {
        CHECK(packet, "inconcistency error: null argument");

        return packet->stream_index == m_stream->index;
    }

    bool Stream::isPassive() const
    {
        return false;
    }

    void Stream::setStatus(Status status)
    {
        m_status = status;
    }

    void Stream::didPlay(const Timer& timer, Status previousStatus)
    {
        setStatus(Playing);
    }

    void Stream::didPause(const Timer& timer, Status previousStatus)
    {
        setStatus(Paused);
    }

    void Stream::didStop(const Timer& timer, Status previousStatus)
    {
        setStatus(Stopped);
    }

    bool Stream::didSeek(const Timer& timer, sf::Time oldPosition)
    {
        if (timer.getOffset() != sf::Time::Zero)
            return fastForward(timer.getOffset());

        return true;
    }

    bool Stream::hasPackets()
    {
        return !m_packetList.empty();
    }
}
<commit_msg>Fix mistake in pull request<commit_after>
/*
 *  Stream.cpp
 *  sfeMovie project
 *
 *  Copyright (C) 2010-2015 Lucas Soltic
 *  lucas.soltic@orange.fr
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 */

extern "C"
{
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
}

#include "Stream.hpp"
#include "Utilities.hpp"
#include "TimerPriorities.hpp"
#include <cassert>
#include <iostream>
#include <stdexcept>

namespace sfe
{
    std::string Stream::AVStreamDescription(AVStream* stream)
    {
        return std::string("'" + std::string(av_get_media_type_string(stream->codec->codec_type))
                           + "/" + avcodec_get_name(stream->codec->codec_id) + "' stream @ " + s(stream));
    }

    Stream::Stream(AVFormatContext*& formatCtx, AVStream*& stream, DataSource& dataSource, std::shared_ptr<Timer> timer) :
    m_formatCtx(formatCtx),
    m_stream(stream),
    m_dataSource(dataSource),
    m_timer(timer),
    m_codec(nullptr),
    m_streamID(-1),
    m_packetList(),
    m_status(Stopped),
    m_readerMutex()
    {
        CHECK(stream, "Stream::Stream() - invalid stream argument");
        CHECK(timer, "Inconcistency error: null timer");
        int err = 0;

        m_stream = stream;
        m_streamID = stream->index;
        CHECK(m_stream, "Inconcistency error: null stream")
        CHECK(m_streamID >= 0, "Inconcistency error: invalid stream id");

        // Get the decoder
        m_codec = avcodec_find_decoder(m_stream->codec->codec_id);
        CHECK(m_codec, "Stream() - no decoder for " + std::string(avcodec_get_name(m_stream->codec->codec_id)) + " codec");

        // Load the codec
        err = avcodec_open2(m_stream->codec, m_codec, nullptr);
        CHECK0(err, "Stream() - unable to load decoder for codec " + std::string(avcodec_get_name(m_stream->codec->codec_id)));

        AVDictionaryEntry* entry = av_dict_get(m_stream->metadata, "language", nullptr, 0);
        if (entry)
        {
            m_language = entry->value;
        }
    }

    Stream::~Stream()
    {
        disconnect();
        flushBuffers();

        if (m_formatCtx && m_stream && m_stream->codec)
        {
            avcodec_close(m_stream->codec);
        }
    }

    void Stream::connect()
    {
        if (isPassive())
            m_timer->addObserver(*this, PassiveStreamTimerPriority);
        else
            m_timer->addObserver(*this, ActiveStreamTimerPriority);
    }

    void Stream::disconnect()
    {
        m_timer->removeObserver(*this);
    }

    void Stream::pushEncodedData(AVPacket* packet)
    {
        CHECK(packet, "invalid argument");
        sf::Lock l(m_readerMutex);
        m_packetList.push_back(packet);
    }

    void Stream::prependEncodedData(AVPacket* packet)
    {
        CHECK(packet, "invalid argument");
        sf::Lock l(m_readerMutex);
        m_packetList.push_front(packet);
    }

    AVPacket* Stream::popEncodedData()
    {
        AVPacket* result = nullptr;
        sf::Lock l(m_readerMutex);

        if (m_packetList.empty() && !isPassive())
        {
            m_dataSource.requestMoreData(*this);
        }

        if (!m_packetList.empty())
        {
            result = m_packetList.front();
            m_packetList.pop_front();
        }
        else
        {
            if (m_stream->codec->codec->capabilities & CODEC_CAP_DELAY)
            {
                AVPacket* flushPacket = (AVPacket*)av_malloc(sizeof(*flushPacket));
                av_init_packet(flushPacket);
                flushPacket->data = nullptr;
                flushPacket->size = 0;
                result = flushPacket;

                sfeLogDebug("Sending flush packet: " + mediaTypeToString(getStreamKind()));
            }
        }

        return result;
    }

    void Stream::flushBuffers()
    {
        sf::Lock l(m_readerMutex);
        if (getStatus() == Playing)
        {
            sfeLogWarning("packets flushed while the stream is still playing");
        }

        if (m_formatCtx && m_stream)
            avcodec_flush_buffers(m_stream->codec);

        AVPacket* pkt = nullptr;

        while (!m_packetList.empty())
        {
            pkt = m_packetList.front();
            m_packetList.pop_front();

            av_free_packet(pkt);
            av_free(pkt);
        }
    }

    bool Stream::needsMoreData() const
    {
        return m_packetList.size() < 10;
    }

    MediaType Stream::getStreamKind() const
    {
        return Unknown;
    }

    Status Stream::getStatus() const
    {
        return m_status;
    }

    std::string Stream::getLanguage() const
    {
        return m_language;
    }

    sf::Time Stream::computeEncodedPosition()
    {
        if (m_packetList.empty())
        {
            m_dataSource.requestMoreData(*this);
        }

        if (m_packetList.empty())
        {
            return sf::Time::Zero;
        }
        else
        {
            sf::Lock l(m_readerMutex);
            AVPacket* packet = m_packetList.front();
            CHECK(packet, "internal inconcistency");

            int64_t timestamp = -424242;

            if (packet->dts != AV_NOPTS_VALUE)
            {
                timestamp = packet->dts;
            }
            else if (packet->pts != AV_NOPTS_VALUE)
            {
                int64_t startTime = m_stream->start_time != AV_NOPTS_VALUE ? m_stream->start_time : 0;
                timestamp = packet->pts - startTime;
            }

            AVRational seconds = av_mul_q(av_make_q(timestamp, 1), m_stream->time_base);
            return sf::milliseconds(1000 * av_q2d(seconds));
        }
    }

    sf::Time Stream::packetDuration(const AVPacket* packet) const
    {
        CHECK(packet, "inconcistency error: null packet");
        CHECK(packet->stream_index == m_streamID, "Asking for duration of a packet for a different stream!");

        if (packet->duration != 0)
        {
            AVRational seconds = av_mul_q(av_make_q(packet->duration, 1), m_stream->time_base);
            return sf::seconds(av_q2d(seconds));
        }
        else
        {
            return sf::seconds(1. / av_q2d(av_guess_frame_rate(m_formatCtx, m_stream, nullptr)));
        }
    }

    std::string Stream::description() const
    {
        return AVStreamDescription(m_stream);
    }

    bool Stream::canUsePacket(AVPacket* packet) const
    {
        CHECK(packet, "inconcistency error: null argument");

        return packet->stream_index == m_stream->index;
    }

    bool Stream::isPassive() const
    {
        return false;
    }

    void Stream::setStatus(Status status)
    {
        m_status = status;
    }

    void Stream::didPlay(const Timer& timer, Status previousStatus)
    {
        setStatus(Playing);
    }

    void Stream::didPause(const Timer& timer, Status previousStatus)
    {
        setStatus(Paused);
    }

    void Stream::didStop(const Timer& timer, Status previousStatus)
    {
        setStatus(Stopped);
    }

    bool Stream::didSeek(const Timer& timer, sf::Time oldPosition)
    {
        if (timer.getOffset() != sf::Time::Zero)
            return fastForward(timer.getOffset());

        return true;
    }

    bool Stream::hasPackets()
    {
        return !m_packetList.empty();
    }
}
<|endoftext|>