<commit_before>

#include <iostream>
#include <string.h>
#include <stdlib.h>
#include <errno.h>

#include "include/types.h"
#include "objclass/objclass.h"

#include "include/rbd_types.h"

CLS_VER(1,2)
CLS_NAME(rbd)

cls_handle_t h_class;
cls_method_handle_t h_snapshots_list;
cls_method_handle_t h_snapshot_add;
cls_method_handle_t h_snapshot_remove;
cls_method_handle_t h_snapshot_revert;
cls_method_handle_t h_assign_bid;

static int snap_read_header(cls_method_context_t hctx, bufferlist& bl)
{
  unsigned snap_count = 0;
  uint64_t snap_names_len = 0;
  int rc;
  struct rbd_obj_header_ondisk *header;

  cls_log("snapshots_list");

  while (1) {
    int len = sizeof(*header) +
      snap_count * sizeof(struct rbd_obj_snap_ondisk) +
      snap_names_len;

    rc = cls_cxx_read(hctx, 0, len, &bl);
    if (rc < 0)
      return rc;

    header = (struct rbd_obj_header_ondisk *)bl.c_str();

    if ((snap_count != header->snap_count) ||
        (snap_names_len != header->snap_names_len)) {
      snap_count = header->snap_count;
      snap_names_len = header->snap_names_len;
      bl.clear();
      continue;
    }
    break;
  }

  return 0;
}

int snapshots_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();
  bufferptr p(header->snap_names_len);
  char *buf = (char *)header;
  char *name = buf + sizeof(*header) + header->snap_count * sizeof(struct rbd_obj_snap_ondisk);
  char *end = name + header->snap_names_len;
  memcpy(p.c_str(),
         buf + sizeof(*header) + header->snap_count * sizeof(struct rbd_obj_snap_ondisk),
         header->snap_names_len);

  ::encode(header->snap_seq, *out);
  ::encode(header->snap_count, *out);

  for (unsigned i = 0; i < header->snap_count; i++) {
    string s = name;
    ::encode(header->snaps[i].id, *out);
    ::encode(header->snaps[i].image_size, *out);
    ::encode(s, *out);

    name += strlen(name) + 1;
    if (name > end)
      return -EIO;
  }

  return 0;
}

int snapshot_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int len = snaps_id_ofs;
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  uint64_t snap_id;

  try {
    ::decode(s, iter);
    ::decode(snap_id, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  const char *cur_snap_name;
  for (cur_snap_name = snap_names; cur_snap_name < end; cur_snap_name += strlen(cur_snap_name) + 1) {
    if (strncmp(cur_snap_name, snap_name, end - cur_snap_name) == 0)
      return -EEXIST;
  }
  if (cur_snap_name > end)
    return -EIO;

  int snap_name_len = strlen(snap_name);

  bufferptr new_names_bp(header->snap_names_len + snap_name_len + 1);
  bufferptr new_snaps_bp(sizeof(*new_snaps) * (header->snap_count + 1));

  /* copy snap names and append to new snap name */
  char *new_snap_names = new_names_bp.c_str();
  strcpy(new_snap_names, snap_name);
  memcpy(new_snap_names + snap_name_len + 1, snap_names, header->snap_names_len);

  /* append new snap id */
  new_snaps = (struct rbd_obj_snap_ondisk *)new_snaps_bp.c_str();
  memcpy(new_snaps + 1, header->snaps, sizeof(*new_snaps) * header->snap_count);

  header->snap_count = header->snap_count + 1;
  header->snap_names_len = header->snap_names_len + snap_name_len + 1;
  header->snap_seq = snap_id;

  new_snaps[0].id = snap_id;
  new_snaps[0].image_size = header->image_size;

  len += sizeof(*new_snaps) * header->snap_count + header->snap_names_len;

  memcpy(header_bp.c_str(), header, sizeof(*header));

  newbl.push_back(header_bp);
  newbl.push_back(new_snaps_bp);
  newbl.push_back(new_names_bp);

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  return 0;
}

int snapshot_revert(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  int i;
  bool found = false;
  struct rbd_obj_snap_ondisk snap;

  try {
    ::decode(s, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  for (i = 0; snap_names < end; i++) {
    if (strcmp(snap_names, snap_name) == 0) {
      snap = header->snaps[i];
      found = true;
      break;
    }
    snap_names += strlen(snap_names) + 1;
  }
  if (!found) {
    CLS_LOG("couldn't find snap %s\n",snap_name);
    return -ENOENT;
  }

  header->image_size = snap.image_size;
  header->snap_seq = header->snap_seq + 1;

  snap_names += strlen(snap_names) + 1;
  i++;

  header->snap_count = header->snap_count - i;
  bufferptr new_names_bp(end - snap_names);
  bufferptr new_snaps_bp(sizeof(header->snaps[0]) * header->snap_count);

  memcpy(header_bp.c_str(), header, sizeof(*header));
  newbl.push_back(header_bp);

  if (header->snap_count) {
    memcpy(new_snaps_bp.c_str(), header->snaps + i, sizeof(header->snaps[0]) * header->snap_count);
    memcpy(new_names_bp.c_str(), snap_names, end - snap_names);
    newbl.push_back(new_snaps_bp);
    newbl.push_back(new_names_bp);
  }

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  ::encode(snap.id, *out);

  return out->length();
}

int snapshot_remove(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *orig_names = snap_names;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  unsigned i;
  bool found = false;
  struct rbd_obj_snap_ondisk snap;

  try {
    ::decode(s, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  for (i = 0; snap_names < end; i++) {
    if (strcmp(snap_names, snap_name) == 0) {
      snap = header->snaps[i];
      found = true;
      break;
    }
    snap_names += strlen(snap_names) + 1;
  }
  if (!found) {
    CLS_LOG("couldn't find snap %s\n",snap_name);
    return -ENOENT;
  }

  snap_names += s.length() + 1;

  header->snap_names_len  = header->snap_names_len - (s.length() + 1);
  header->snap_count = header->snap_count - 1;

  bufferptr new_names_bp(header->snap_names_len);
  bufferptr new_snaps_bp(sizeof(header->snaps[0]) * header->snap_count);

  memcpy(header_bp.c_str(), header, sizeof(*header));
  newbl.push_back(header_bp);

  if (header->snap_count) {
    if (i > 0) {
      memcpy(new_snaps_bp.c_str(), header->snaps, sizeof(header->snaps[0]) * i);
      memcpy(new_names_bp.c_str(), header->snaps, snap_names - orig_names);
    }
    snap_names += s.length() + 1;

    if (i < header->snap_count) {
      memcpy(new_snaps_bp.c_str(), header->snaps, sizeof(header->snaps[0]) * i);
      memcpy(new_names_bp.c_str(), snap_names , end - snap_names);
    }
  }

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  return 0;

}

/* assign block id. This method should be called on the rbd_info object */
int rbd_assign_bid(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  struct rbd_info info;
  int rc;
  bufferlist bl;

  rc = cls_cxx_read(hctx, 0, sizeof(info), &bl);
  if (rc < 0 && rc != -EEXIST)
    return rc;

  if (rc && rc < (int)sizeof(info)) {
    CLS_LOG("bad rbd_info object, read %d bytes, expected %d", rc, sizeof(info));
    return -EIO;
  }

  uint64_t max_id;
  if (rc) {
    memcpy(&info, bl.c_str(), sizeof(info));
    max_id = info.max_id + 1;
    info.max_id = max_id;
  } else {
    memset(&info, 0, sizeof(info));
    max_id = 0;
  }

  bufferlist newbl;
  bufferptr bp(sizeof(info));
  memcpy(bp.c_str(), &info, sizeof(info));
  newbl.push_back(bp);
  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0) {
    CLS_LOG("error writing rbd_info, got rc=%d", rc);
    return rc;
  }

  ::encode(max_id, *out);

  return out->length();
}

void __cls_init()
{
  CLS_LOG("Loaded rbd class!");

  cls_register("rbd", &h_class);
  cls_register_cxx_method(h_class, "snap_list", CLS_METHOD_RD | CLS_METHOD_PUBLIC, snapshots_list, &h_snapshots_list);
  cls_register_cxx_method(h_class, "snap_add", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_add, &h_snapshot_add);
  cls_register_cxx_method(h_class, "snap_remove", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_remove, &h_snapshot_remove);
  cls_register_cxx_method(h_class, "snap_revert", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_revert, &h_snapshot_revert);

  /* assign a unique block id for rbd blocks */
  cls_register_cxx_method(h_class, "assign_bid", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rbd_assign_bid, &h_assign_bid);

  return;
}

<commit_msg>rbd: fix rbd snap rm class handling<commit_after>

#include <iostream>
#include <string.h>
#include <stdlib.h>
#include <errno.h>

#include "include/types.h"
#include "objclass/objclass.h"

#include "include/rbd_types.h"

CLS_VER(1,3)
CLS_NAME(rbd)

cls_handle_t h_class;
cls_method_handle_t h_snapshots_list;
cls_method_handle_t h_snapshot_add;
cls_method_handle_t h_snapshot_remove;
cls_method_handle_t h_snapshot_revert;
cls_method_handle_t h_assign_bid;

static int snap_read_header(cls_method_context_t hctx, bufferlist& bl)
{
  unsigned snap_count = 0;
  uint64_t snap_names_len = 0;
  int rc;
  struct rbd_obj_header_ondisk *header;

  cls_log("snapshots_list");

  while (1) {
    int len = sizeof(*header) +
      snap_count * sizeof(struct rbd_obj_snap_ondisk) +
      snap_names_len;

    rc = cls_cxx_read(hctx, 0, len, &bl);
    if (rc < 0)
      return rc;

    header = (struct rbd_obj_header_ondisk *)bl.c_str();

    if ((snap_count != header->snap_count) ||
        (snap_names_len != header->snap_names_len)) {
      snap_count = header->snap_count;
      snap_names_len = header->snap_names_len;
      bl.clear();
      continue;
    }
    break;
  }

  return 0;
}

int snapshots_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();
  bufferptr p(header->snap_names_len);
  char *buf = (char *)header;
  char *name = buf + sizeof(*header) + header->snap_count * sizeof(struct rbd_obj_snap_ondisk);
  char *end = name + header->snap_names_len;
  memcpy(p.c_str(),
         buf + sizeof(*header) + header->snap_count * sizeof(struct rbd_obj_snap_ondisk),
         header->snap_names_len);

  ::encode(header->snap_seq, *out);
  ::encode(header->snap_count, *out);

  for (unsigned i = 0; i < header->snap_count; i++) {
    string s = name;
    ::encode(header->snaps[i].id, *out);
    ::encode(header->snaps[i].image_size, *out);
    ::encode(s, *out);

    name += strlen(name) + 1;
    if (name > end)
      return -EIO;
  }

  return 0;
}

int snapshot_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int len = snaps_id_ofs;
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  uint64_t snap_id;

  try {
    ::decode(s, iter);
    ::decode(snap_id, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  const char *cur_snap_name;
  for (cur_snap_name = snap_names; cur_snap_name < end; cur_snap_name += strlen(cur_snap_name) + 1) {
    if (strncmp(cur_snap_name, snap_name, end - cur_snap_name) == 0)
      return -EEXIST;
  }
  if (cur_snap_name > end)
    return -EIO;

  int snap_name_len = strlen(snap_name);

  bufferptr new_names_bp(header->snap_names_len + snap_name_len + 1);
  bufferptr new_snaps_bp(sizeof(*new_snaps) * (header->snap_count + 1));

  /* copy snap names and append to new snap name */
  char *new_snap_names = new_names_bp.c_str();
  strcpy(new_snap_names, snap_name);
  memcpy(new_snap_names + snap_name_len + 1, snap_names, header->snap_names_len);

  /* append new snap id */
  new_snaps = (struct rbd_obj_snap_ondisk *)new_snaps_bp.c_str();
  memcpy(new_snaps + 1, header->snaps, sizeof(*new_snaps) * header->snap_count);

  header->snap_count = header->snap_count + 1;
  header->snap_names_len = header->snap_names_len + snap_name_len + 1;
  header->snap_seq = snap_id;

  new_snaps[0].id = snap_id;
  new_snaps[0].image_size = header->image_size;

  len += sizeof(*new_snaps) * header->snap_count + header->snap_names_len;

  memcpy(header_bp.c_str(), header, sizeof(*header));

  newbl.push_back(header_bp);
  newbl.push_back(new_snaps_bp);
  newbl.push_back(new_names_bp);

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  return 0;
}

int snapshot_revert(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  int i;
  bool found = false;
  struct rbd_obj_snap_ondisk snap;

  try {
    ::decode(s, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  for (i = 0; snap_names < end; i++) {
    if (strcmp(snap_names, snap_name) == 0) {
      snap = header->snaps[i];
      found = true;
      break;
    }
    snap_names += strlen(snap_names) + 1;
  }
  if (!found) {
    CLS_LOG("couldn't find snap %s\n",snap_name);
    return -ENOENT;
  }

  header->image_size = snap.image_size;
  header->snap_seq = header->snap_seq + 1;

  snap_names += strlen(snap_names) + 1;
  i++;

  header->snap_count = header->snap_count - i;
  bufferptr new_names_bp(end - snap_names);
  bufferptr new_snaps_bp(sizeof(header->snaps[0]) * header->snap_count);

  memcpy(header_bp.c_str(), header, sizeof(*header));
  newbl.push_back(header_bp);

  if (header->snap_count) {
    memcpy(new_snaps_bp.c_str(), header->snaps + i, sizeof(header->snaps[0]) * header->snap_count);
    memcpy(new_names_bp.c_str(), snap_names, end - snap_names);
    newbl.push_back(new_snaps_bp);
    newbl.push_back(new_names_bp);
  }

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  ::encode(snap.id, *out);

  return out->length();
}

int snapshot_remove(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rbd_obj_header_ondisk *header;
  bufferlist newbl;
  bufferptr header_bp(sizeof(*header));
  struct rbd_obj_snap_ondisk *new_snaps;

  int rc = snap_read_header(hctx, bl);
  if (rc < 0)
    return rc;

  header = (struct rbd_obj_header_ondisk *)bl.c_str();

  int snaps_id_ofs = sizeof(*header);
  int names_ofs = snaps_id_ofs + sizeof(*new_snaps) * header->snap_count;
  const char *snap_name;
  const char *snap_names = ((char *)header) + names_ofs;
  const char *orig_names = snap_names;
  const char *end = snap_names + header->snap_names_len;
  bufferlist::iterator iter = in->begin();
  string s;
  unsigned i;
  bool found = false;
  struct rbd_obj_snap_ondisk snap;

  try {
    ::decode(s, iter);
  } catch (const buffer::error &err) {
    return -EINVAL;
  }
  snap_name = s.c_str();

  for (i = 0; snap_names < end; i++) {
    if (strcmp(snap_names, snap_name) == 0) {
      snap = header->snaps[i];
      found = true;
      break;
    }
    snap_names += strlen(snap_names) + 1;
  }
  if (!found) {
    CLS_LOG("couldn't find snap %s\n",snap_name);
    return -ENOENT;
  }

  header->snap_names_len  = header->snap_names_len - (s.length() + 1);
  header->snap_count = header->snap_count - 1;

  bufferptr new_names_bp(header->snap_names_len);
  bufferptr new_snaps_bp(sizeof(header->snaps[0]) * header->snap_count);

  memcpy(header_bp.c_str(), header, sizeof(*header));
  newbl.push_back(header_bp);

  if (header->snap_count) {
    int snaps_len = 0;
    int names_len = 0;
    CLS_LOG("i=%d\n", i);
    if (i > 0) {
      snaps_len = sizeof(header->snaps[0]) * i;
      names_len =  snap_names - orig_names;
      memcpy(new_snaps_bp.c_str(), header->snaps, snaps_len);
      memcpy(new_names_bp.c_str(), orig_names, names_len);
    }
    snap_names += s.length() + 1;

    if (i < header->snap_count) {
      memcpy(new_snaps_bp.c_str() + snaps_len,
             header->snaps + i + 1,
             sizeof(header->snaps[0]) * (header->snap_count - i));
      memcpy(new_names_bp.c_str() + names_len, snap_names , end - snap_names);
    }
    newbl.push_back(new_snaps_bp);
    newbl.push_back(new_names_bp);
  }

  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0)
    return rc;

  return 0;

}

/* assign block id. This method should be called on the rbd_info object */
int rbd_assign_bid(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  struct rbd_info info;
  int rc;
  bufferlist bl;

  rc = cls_cxx_read(hctx, 0, sizeof(info), &bl);
  if (rc < 0 && rc != -EEXIST)
    return rc;

  if (rc && rc < (int)sizeof(info)) {
    CLS_LOG("bad rbd_info object, read %d bytes, expected %d", rc, sizeof(info));
    return -EIO;
  }

  uint64_t max_id;
  if (rc) {
    memcpy(&info, bl.c_str(), sizeof(info));
    max_id = info.max_id + 1;
    info.max_id = max_id;
  } else {
    memset(&info, 0, sizeof(info));
    max_id = 0;
  }

  bufferlist newbl;
  bufferptr bp(sizeof(info));
  memcpy(bp.c_str(), &info, sizeof(info));
  newbl.push_back(bp);
  rc = cls_cxx_write_full(hctx, &newbl);
  if (rc < 0) {
    CLS_LOG("error writing rbd_info, got rc=%d", rc);
    return rc;
  }

  ::encode(max_id, *out);

  return out->length();
}

void __cls_init()
{
  CLS_LOG("Loaded rbd class!");

  cls_register("rbd", &h_class);
  cls_register_cxx_method(h_class, "snap_list", CLS_METHOD_RD | CLS_METHOD_PUBLIC, snapshots_list, &h_snapshots_list);
  cls_register_cxx_method(h_class, "snap_add", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_add, &h_snapshot_add);
  cls_register_cxx_method(h_class, "snap_remove", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_remove, &h_snapshot_remove);
  cls_register_cxx_method(h_class, "snap_revert", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, snapshot_revert, &h_snapshot_revert);

  /* assign a unique block id for rbd blocks */
  cls_register_cxx_method(h_class, "assign_bid", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rbd_assign_bid, &h_assign_bid);

  return;
}

<|endoftext|>


<commit_before>#include <iostream>

#include <string.h>
#include <stdlib.h>
#include <errno.h>

#include "include/types.h"
#include "include/utime.h"
#include "objclass/objclass.h"
#include "rgw/rgw_cls_api.h"
#include "common/Clock.h"

#include "global/global_context.h"

CLS_VER(1,0)
CLS_NAME(rgw)

cls_handle_t h_class;
cls_method_handle_t h_rgw_bucket_init_index;
cls_method_handle_t h_rgw_bucket_list;
cls_method_handle_t h_rgw_bucket_prepare_op;
cls_method_handle_t h_rgw_bucket_complete_op;

#define ROUND_BLOCK_SIZE 4096

static uint64_t get_rounded_size(uint64_t size)
{
  return (size + ROUND_BLOCK_SIZE - 1) & ~(ROUND_BLOCK_SIZE - 1);
}

static int read_bucket_dir(cls_method_context_t hctx, struct rgw_bucket_dir& dir)
{
  bufferlist bl;

  uint64_t size;
  int rc = cls_cxx_stat(hctx, &size, NULL);
  if (rc < 0)
    return rc;

  rc = cls_cxx_read(hctx, 0, size, &bl);
  if (rc < 0)
    return rc;

  try {
    bufferlist::iterator iter = bl.begin();
    ::decode(dir, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: read_bucket_dir(): failed to decode buffer\n");
    return -EIO;
  }

  return 0;
}

static int write_bucket_dir(cls_method_context_t hctx, struct rgw_bucket_dir& dir)
{
  bufferlist bl;

  ::encode(dir, bl);

  int rc = cls_cxx_write_full(hctx, &bl);
  return rc;
}

int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  bufferlist::iterator iter = in->begin();

  struct rgw_cls_list_op op;

  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_list(): failed to decode request\n");
    return -EINVAL;
  }

  struct rgw_cls_list_ret ret;
  struct rgw_bucket_dir& new_dir = ret.dir;
  new_dir.header = dir.header;
  std::map<string, struct rgw_bucket_dir_entry>& m = new_dir.m;
  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.upper_bound(op.start_obj);
  uint32_t i;
  for (i = 0; i != op.num_entries && miter != dir.m.end(); ++i, ++miter) {
    m[miter->first] = miter->second;
  }

  ret.is_truncated = (miter != dir.m.end());

  ::encode(ret, *out);

  return 0;
}

int rgw_bucket_init_index(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  bufferlist::iterator iter;

  uint64_t size;
  int rc = cls_cxx_stat(hctx, &size, NULL);
  if (rc < 0)
    return rc;
  if (size != 0) {
    CLS_LOG("ERROR: index already initialized\n");
    return -EINVAL;
  }

  rgw_bucket_dir dir;
  rc = write_bucket_dir(hctx, dir);

  return rc;
}

int rgw_bucket_prepare_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  rgw_cls_obj_prepare_op op;

  bufferlist::iterator iter = in->begin();
  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_prepare_op(): failed to decode request\n");
    return -EINVAL;
  }
  CLS_LOG("rgw_bucket_prepare_op(): request: op=%d name=%s tag=%s\n", op.op, op.name.c_str(), op.tag.c_str());

  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.find(op.name);
  struct rgw_bucket_dir_entry *entry = NULL;

  if (miter != dir.m.end()) {
    entry = &miter->second;
  } else {
    entry = &dir.m[op.name];
    entry->name = op.name;
    entry->epoch = 0;
    entry->exists = false;
  }

  if (op.tag.empty()) {
    CLS_LOG("ERROR: tag is empty\n");
    return -EINVAL;
  }

  struct rgw_bucket_pending_info& info = entry->pending_map[op.tag];
  info.timestamp = ceph_clock_now(g_ceph_context);
  info.state = CLS_RGW_STATE_PENDING_MODIFY;
  info.op = op.op;

  entry->pending_map[op.tag] = info;

  rc = write_bucket_dir(hctx, dir);

  return rc;
}

int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  CLS_LOG("rgw_bucket_complete_op(): dir.m.size()=%lld", dir.m.size());

  rgw_cls_obj_complete_op op;

  bufferlist::iterator iter = in->begin();
  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_complete_op(): failed to decode request\n");
    return -EINVAL;
  }
  CLS_LOG("rgw_bucket_complete_op(): request: op=%d name=%s epoch=%lld tag=%s\n", op.op, op.name.c_str(), op.epoch, op.tag.c_str());
  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.find(op.name);
  struct rgw_bucket_dir_entry *entry = NULL;

  CLS_LOG("rgw_bucket_modify(): dir.m.size()=%lld", dir.m.size());

  if (miter != dir.m.end()) {
    entry = &miter->second;
    CLS_LOG("rgw_bucket_complete_op(): existing entry: epoch=%lld\n", entry->epoch);
    if (op.epoch <= entry->epoch) {
      CLS_LOG("rgw_bucket_complete_op(): skipping request, old epoch\n");
      return 0;
    }

    if (entry.exists) {
      struct rgw_bucket_category_stats& stats = dir.header.stats[entry->meta.category];
      stats.num_entries--;
      stats.total_size -= entry->meta.size;
      stats.total_size_rounded -= get_rounded_size(entry->meta.size);
    }
  } else {
    entry = &dir.m[op.name];
    entry->name = op.name;
    entry->epoch = op.epoch;
    entry->meta = op.meta;
  }

  if (op.tag.size()) {
    map<string, struct rgw_bucket_pending_info>::iterator pinter = entry->pending_map.find(op.tag);
    if (pinter == entry->pending_map.end()) {
      CLS_LOG("ERROR: couldn't find tag for pending operation\n");
      return -EINVAL;
    }
    entry->pending_map.erase(pinter);
  }

  switch (op.op) {
  case CLS_RGW_OP_DEL:
    if (miter != dir.m.end()) {
      if (!entry->pending_map.size())
        dir.m.erase(miter);
      else
        entry->exists = false;
    } else {
      return -ENOENT;
    }
    break;
  case CLS_RGW_OP_ADD:
    {
      struct rgw_bucket_dir_entry_meta& meta = op.meta;
      struct rgw_bucket_category_stats& stats = dir.header.stats[meta.category];
      entry->meta = meta;
      entry->name = op.name;
      entry->epoch = op.epoch;
      entry->exists = true;
      stats.num_entries++;
      stats.total_size += meta.size;
      stats.total_size_rounded += get_rounded_size(meta.size);
    }
    break;
  }

  rc = write_bucket_dir(hctx, dir);

  return rc;
}

void __cls_init()
{
  CLS_LOG("Loaded rgw class!");

  cls_register("rgw", &h_class);
  cls_register_cxx_method(h_class, "bucket_init_index", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_init_index, &h_rgw_bucket_init_index);
  cls_register_cxx_method(h_class, "bucket_list", CLS_METHOD_RD | CLS_METHOD_PUBLIC, rgw_bucket_list, &h_rgw_bucket_list);
  cls_register_cxx_method(h_class, "bucket_prepare_op", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_prepare_op, &h_rgw_bucket_prepare_op);
  cls_register_cxx_method(h_class, "bucket_complete_op", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_complete_op, &h_rgw_bucket_complete_op);

  return;
}

<commit_msg>rgw: fix compilation<commit_after>#include <iostream>

#include <string.h>
#include <stdlib.h>
#include <errno.h>

#include "include/types.h"
#include "include/utime.h"
#include "objclass/objclass.h"
#include "rgw/rgw_cls_api.h"
#include "common/Clock.h"

#include "global/global_context.h"

CLS_VER(1,0)
CLS_NAME(rgw)

cls_handle_t h_class;
cls_method_handle_t h_rgw_bucket_init_index;
cls_method_handle_t h_rgw_bucket_list;
cls_method_handle_t h_rgw_bucket_prepare_op;
cls_method_handle_t h_rgw_bucket_complete_op;

#define ROUND_BLOCK_SIZE 4096

static uint64_t get_rounded_size(uint64_t size)
{
  return (size + ROUND_BLOCK_SIZE - 1) & ~(ROUND_BLOCK_SIZE - 1);
}

static int read_bucket_dir(cls_method_context_t hctx, struct rgw_bucket_dir& dir)
{
  bufferlist bl;

  uint64_t size;
  int rc = cls_cxx_stat(hctx, &size, NULL);
  if (rc < 0)
    return rc;

  rc = cls_cxx_read(hctx, 0, size, &bl);
  if (rc < 0)
    return rc;

  try {
    bufferlist::iterator iter = bl.begin();
    ::decode(dir, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: read_bucket_dir(): failed to decode buffer\n");
    return -EIO;
  }

  return 0;
}

static int write_bucket_dir(cls_method_context_t hctx, struct rgw_bucket_dir& dir)
{
  bufferlist bl;

  ::encode(dir, bl);

  int rc = cls_cxx_write_full(hctx, &bl);
  return rc;
}

int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  bufferlist::iterator iter = in->begin();

  struct rgw_cls_list_op op;

  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_list(): failed to decode request\n");
    return -EINVAL;
  }

  struct rgw_cls_list_ret ret;
  struct rgw_bucket_dir& new_dir = ret.dir;
  new_dir.header = dir.header;
  std::map<string, struct rgw_bucket_dir_entry>& m = new_dir.m;
  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.upper_bound(op.start_obj);
  uint32_t i;
  for (i = 0; i != op.num_entries && miter != dir.m.end(); ++i, ++miter) {
    m[miter->first] = miter->second;
  }

  ret.is_truncated = (miter != dir.m.end());

  ::encode(ret, *out);

  return 0;
}

int rgw_bucket_init_index(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  bufferlist::iterator iter;

  uint64_t size;
  int rc = cls_cxx_stat(hctx, &size, NULL);
  if (rc < 0)
    return rc;
  if (size != 0) {
    CLS_LOG("ERROR: index already initialized\n");
    return -EINVAL;
  }

  rgw_bucket_dir dir;
  rc = write_bucket_dir(hctx, dir);

  return rc;
}

int rgw_bucket_prepare_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  rgw_cls_obj_prepare_op op;

  bufferlist::iterator iter = in->begin();
  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_prepare_op(): failed to decode request\n");
    return -EINVAL;
  }
  CLS_LOG("rgw_bucket_prepare_op(): request: op=%d name=%s tag=%s\n", op.op, op.name.c_str(), op.tag.c_str());

  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.find(op.name);
  struct rgw_bucket_dir_entry *entry = NULL;

  if (miter != dir.m.end()) {
    entry = &miter->second;
  } else {
    entry = &dir.m[op.name];
    entry->name = op.name;
    entry->epoch = 0;
    entry->exists = false;
  }

  if (op.tag.empty()) {
    CLS_LOG("ERROR: tag is empty\n");
    return -EINVAL;
  }

  struct rgw_bucket_pending_info& info = entry->pending_map[op.tag];
  info.timestamp = ceph_clock_now(g_ceph_context);
  info.state = CLS_RGW_STATE_PENDING_MODIFY;
  info.op = op.op;

  entry->pending_map[op.tag] = info;

  rc = write_bucket_dir(hctx, dir);

  return rc;
}

int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
{
  bufferlist bl;
  struct rgw_bucket_dir dir;
  int rc = read_bucket_dir(hctx, dir);
  if (rc < 0)
    return rc;

  CLS_LOG("rgw_bucket_complete_op(): dir.m.size()=%lld", dir.m.size());

  rgw_cls_obj_complete_op op;

  bufferlist::iterator iter = in->begin();
  try {
    ::decode(op, iter);
  } catch (buffer::error& err) {
    CLS_LOG("ERROR: rgw_bucket_complete_op(): failed to decode request\n");
    return -EINVAL;
  }
  CLS_LOG("rgw_bucket_complete_op(): request: op=%d name=%s epoch=%lld tag=%s\n", op.op, op.name.c_str(), op.epoch, op.tag.c_str());
  std::map<string, struct rgw_bucket_dir_entry>::iterator miter = dir.m.find(op.name);
  struct rgw_bucket_dir_entry *entry = NULL;

  CLS_LOG("rgw_bucket_modify(): dir.m.size()=%lld", dir.m.size());

  if (miter != dir.m.end()) {
    entry = &miter->second;
    CLS_LOG("rgw_bucket_complete_op(): existing entry: epoch=%lld\n", entry->epoch);
    if (op.epoch <= entry->epoch) {
      CLS_LOG("rgw_bucket_complete_op(): skipping request, old epoch\n");
      return 0;
    }

    if (entry->exists) {
      struct rgw_bucket_category_stats& stats = dir.header.stats[entry->meta.category];
      stats.num_entries--;
      stats.total_size -= entry->meta.size;
      stats.total_size_rounded -= get_rounded_size(entry->meta.size);
    }
  } else {
    entry = &dir.m[op.name];
    entry->name = op.name;
    entry->epoch = op.epoch;
    entry->meta = op.meta;
  }

  if (op.tag.size()) {
    map<string, struct rgw_bucket_pending_info>::iterator pinter = entry->pending_map.find(op.tag);
    if (pinter == entry->pending_map.end()) {
      CLS_LOG("ERROR: couldn't find tag for pending operation\n");
      return -EINVAL;
    }
    entry->pending_map.erase(pinter);
  }

  switch (op.op) {
  case CLS_RGW_OP_DEL:
    if (miter != dir.m.end()) {
      if (!entry->pending_map.size())
        dir.m.erase(miter);
      else
        entry->exists = false;
    } else {
      return -ENOENT;
    }
    break;
  case CLS_RGW_OP_ADD:
    {
      struct rgw_bucket_dir_entry_meta& meta = op.meta;
      struct rgw_bucket_category_stats& stats = dir.header.stats[meta.category];
      entry->meta = meta;
      entry->name = op.name;
      entry->epoch = op.epoch;
      entry->exists = true;
      stats.num_entries++;
      stats.total_size += meta.size;
      stats.total_size_rounded += get_rounded_size(meta.size);
    }
    break;
  }

  rc = write_bucket_dir(hctx, dir);

  return rc;
}

void __cls_init()
{
  CLS_LOG("Loaded rgw class!");

  cls_register("rgw", &h_class);
  cls_register_cxx_method(h_class, "bucket_init_index", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_init_index, &h_rgw_bucket_init_index);
  cls_register_cxx_method(h_class, "bucket_list", CLS_METHOD_RD | CLS_METHOD_PUBLIC, rgw_bucket_list, &h_rgw_bucket_list);
  cls_register_cxx_method(h_class, "bucket_prepare_op", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_prepare_op, &h_rgw_bucket_prepare_op);
  cls_register_cxx_method(h_class, "bucket_complete_op", CLS_METHOD_RD | CLS_METHOD_WR | CLS_METHOD_PUBLIC, rgw_bucket_complete_op, &h_rgw_bucket_complete_op);

  return;
}

<|endoftext|>


<commit_before>/*
 * Copyright (C) 2006 Tommi Maekitalo
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * As a special exception, you may use this file as part of a free
 * software library without restriction. Specifically, if other files
 * instantiate templates or use macros or inline functions from this
 * file, or you compile this file and link it with other files to
 * produce an executable, this file does not by itself cause the
 * resulting executable to be covered by the GNU General Public
 * License. This exception does not however invalidate any other
 * reasons why the executable file might be covered by the GNU Library
 * General Public License.
 *
 * This program is distributed in the hope that it will be useful, but
 * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
 * NON-INFRINGEMENT.  See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 */

#include <cxxtools/fdstream.h>
#include <cxxtools/syserror.h>
#include <cxxtools/log.h>
#include <algorithm>
#include <unistd.h>
#include <errno.h>

log_define("cxxtools.fdstream")

namespace cxxtools
{
  Fdstreambuf::Fdstreambuf(int fd_, unsigned bufsize_, bool doClose_)
    : fd(fd_),
      doClose(doClose_),
      bufsize(bufsize_),
      ibuffer(0),
      obuffer(0)
  { }

  Fdstreambuf::~Fdstreambuf()
  {
    delete [] ibuffer;
    delete [] obuffer;

    if (doClose)
      ::close(fd);
  }

  void Fdstreambuf::close()
  {
    ::close(fd);
    doClose = false;
  }

  std::streambuf::int_type Fdstreambuf::overflow(std::streambuf::int_type ch)
  {
    log_debug("overflow(" << ch << ')');

    if (pptr() != pbase())
    {
      log_debug("write " << (pptr() - pbase()) << " bytes to fd " << fd);
      ssize_t ret = ::write(fd, pbase(), pptr() - pbase());

      if(ret < 0)
        throw SystemError(errno, "write");

      if (ret == 0)
        return traits_type::eof();
      else
      {
        log_debug(ret << " bytes written to fd " << fd);
        std::copy(pptr(), pptr() + ret, obuffer);
        setp(obuffer + ret, obuffer + bufsize - ret);
      }
    }
    else
    {
      log_debug("initialize outputbuffer");
      if (obuffer == 0)
      {
        log_debug("allocate " << bufsize << " bytes output buffer");
        obuffer = new char[bufsize];
      }

      setp(obuffer, obuffer + bufsize);
    }

    if (ch != traits_type::eof())
    {
      *pptr() = traits_type::to_char_type(ch);
      pbump(1);
    }

    return 0;
  }

  std::streambuf::int_type Fdstreambuf::underflow()
  {
    if (sync() != 0)
      return traits_type::eof();

    if (ibuffer == 0)
    {
      log_debug("allocate " << bufsize << " bytes input buffer");
      ibuffer = new char[bufsize];
    }

    log_debug("read from fd " << fd);
    int ret = ::read(fd, ibuffer, bufsize);

    if(ret < 0)
      throw SystemError(errno, "read");

    if (ret == 0)
      return traits_type::eof();

    log_debug(ret << " bytes read");
    setg(ibuffer, ibuffer, ibuffer + ret);

    return traits_type::to_char_type(*gptr());
  }

  int Fdstreambuf::sync()
  {
    log_debug("sync()");
    if (pptr() != pbase())
    {
      char* p = pbase();
      while (p < pptr())
      {
        log_debug("write " << (pptr() - p) << " bytes to fd " << fd);
        ssize_t ret = ::write(fd, p, pptr() - p);

        if(ret < 0)
          throw SystemError(errno, "write");

        if (ret == 0)
          return traits_type::eof();

        log_debug(ret << " bytes written to fd " << fd);
        p += ret;
      }
    }
    return 0;
  }

}
<commit_msg>retired syserror<commit_after>/*
 * Copyright (C) 2006 Tommi Maekitalo
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * As a special exception, you may use this file as part of a free
 * software library without restriction. Specifically, if other files
 * instantiate templates or use macros or inline functions from this
 * file, or you compile this file and link it with other files to
 * produce an executable, this file does not by itself cause the
 * resulting executable to be covered by the GNU General Public
 * License. This exception does not however invalidate any other
 * reasons why the executable file might be covered by the GNU Library
 * General Public License.
 *
 * This program is distributed in the hope that it will be useful, but
 * is provided AS IS, WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, and
 * NON-INFRINGEMENT.  See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 */

#include <cxxtools/fdstream.h>
#include <cxxtools/systemerror.h>
#include <cxxtools/log.h>
#include <algorithm>
#include <unistd.h>
#include <errno.h>

log_define("cxxtools.fdstream")

namespace cxxtools
{
  Fdstreambuf::Fdstreambuf(int fd_, unsigned bufsize_, bool doClose_)
    : fd(fd_),
      doClose(doClose_),
      bufsize(bufsize_),
      ibuffer(0),
      obuffer(0)
  { }

  Fdstreambuf::~Fdstreambuf()
  {
    delete [] ibuffer;
    delete [] obuffer;

    if (doClose)
      ::close(fd);
  }

  void Fdstreambuf::close()
  {
    ::close(fd);
    doClose = false;
  }

  std::streambuf::int_type Fdstreambuf::overflow(std::streambuf::int_type ch)
  {
    log_debug("overflow(" << ch << ')');

    if (pptr() != pbase())
    {
      log_debug("write " << (pptr() - pbase()) << " bytes to fd " << fd);
      ssize_t ret = ::write(fd, pbase(), pptr() - pbase());

      if(ret < 0)
        throw SystemError(errno, "write");

      if (ret == 0)
        return traits_type::eof();
      else
      {
        log_debug(ret << " bytes written to fd " << fd);
        std::copy(pptr(), pptr() + ret, obuffer);
        setp(obuffer + ret, obuffer + bufsize - ret);
      }
    }
    else
    {
      log_debug("initialize outputbuffer");
      if (obuffer == 0)
      {
        log_debug("allocate " << bufsize << " bytes output buffer");
        obuffer = new char[bufsize];
      }

      setp(obuffer, obuffer + bufsize);
    }

    if (ch != traits_type::eof())
    {
      *pptr() = traits_type::to_char_type(ch);
      pbump(1);
    }

    return 0;
  }

  std::streambuf::int_type Fdstreambuf::underflow()
  {
    if (sync() != 0)
      return traits_type::eof();

    if (ibuffer == 0)
    {
      log_debug("allocate " << bufsize << " bytes input buffer");
      ibuffer = new char[bufsize];
    }

    log_debug("read from fd " << fd);
    int ret = ::read(fd, ibuffer, bufsize);

    if(ret < 0)
      throw SystemError(errno, "read");

    if (ret == 0)
      return traits_type::eof();

    log_debug(ret << " bytes read");
    setg(ibuffer, ibuffer, ibuffer + ret);

    return traits_type::to_char_type(*gptr());
  }

  int Fdstreambuf::sync()
  {
    log_debug("sync()");
    if (pptr() != pbase())
    {
      char* p = pbase();
      while (p < pptr())
      {
        log_debug("write " << (pptr() - p) << " bytes to fd " << fd);
        ssize_t ret = ::write(fd, p, pptr() - p);

        if(ret < 0)
          throw SystemError(errno, "write");

        if (ret == 0)
          return traits_type::eof();

        log_debug(ret << " bytes written to fd " << fd);
        p += ret;
      }
    }
    return 0;
  }

}
<|endoftext|>


<commit_before>// Copyright (c) 2017, WNProject Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE.txt file.

#ifndef __WN_CONSOLE_LOGGER_LINUX_INL__
#define __WN_CONSOLE_LOGGER_LINUX_INL__

#include <android/log.h>

#include "WNLogging/inc/WNConsoleLogger.h"
#include "WNUtilities/inc/WNLoggingData.h"

namespace wn {
namespace logging {

const int32_t s_log_levels[] = {ANDROID_LOG_SILENT, ANDROID_LOG_FATAL,
    ANDROID_LOG_ERROR, ANDROID_LOG_WARN, ANDROID_LOG_WARN, ANDROID_LOG_INFO,
    ANDROID_LOG_DEBUG, ANDROID_LOG_DEFAULT};

template <console_location T_Location>
console_logger<T_Location>::console_logger()
  : m_log_priority(ANDROID_LOG_INFO) {}

template <console_location T_Location>
console_logger<T_Location>::~console_logger() {}

template <console_location T_Location>
void console_logger<T_Location>::flush_buffer(const char* _buffer,
    size_t _buffer_size,
    const color_element* _colors, size_t _num_colors) {
  if (_num_colors) {
    for (size_t i = 0; i < _num_colors; ++i) {
      if (static_cast<uint32_t>(_colors[i].m_level) > 0) {
        m_log_priority = static_cast<uint32_t>(_colors[i].m_level);
      } else {
        const char* end_color = (_num_colors == i + 1)
                                   ? _buffer + _buffer_size
                                   : _colors[i + 1].m_position;
        size_t len = end_color - _colors[i].m_position;
        __android_log_print(s_log_levels[static_cast<uint32_t>(m_log_priority)], wn::utilities::gAndroidLogTag, "%.*s",
            static_cast<int32_t>(len), _colors[i].m_position);
      }
    }
  } else {
    __android_log_print(
        s_log_levels[static_cast<uint32_t>(m_log_priority)], wn::utilities::gAndroidLogTag, "%.*s", (_buffer_size), _buffer);
  }
}

} // namespace logging
} // namespace wn

#endif  //__WN_CONSOLE_LOGGER_LINUX_INL__
<commit_msg>Fixed bug on arm64 platforms for Android<commit_after>// Copyright (c) 2017, WNProject Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE.txt file.

#ifndef __WN_CONSOLE_LOGGER_LINUX_INL__
#define __WN_CONSOLE_LOGGER_LINUX_INL__

#include <android/log.h>

#include "WNLogging/inc/WNConsoleLogger.h"
#include "WNUtilities/inc/WNLoggingData.h"

namespace wn {
namespace logging {

const int32_t s_log_levels[] = {ANDROID_LOG_SILENT, ANDROID_LOG_FATAL,
    ANDROID_LOG_ERROR, ANDROID_LOG_WARN, ANDROID_LOG_WARN, ANDROID_LOG_INFO,
    ANDROID_LOG_DEBUG, ANDROID_LOG_DEFAULT};

template <console_location T_Location>
console_logger<T_Location>::console_logger()
  : m_log_priority(ANDROID_LOG_INFO) {}

template <console_location T_Location>
console_logger<T_Location>::~console_logger() {}

template <console_location T_Location>
void console_logger<T_Location>::flush_buffer(const char* _buffer,
    size_t _buffer_size, const color_element* _colors, size_t _num_colors) {
  if (_num_colors) {
    for (size_t i = 0; i < _num_colors; ++i) {
      if (static_cast<uint32_t>(_colors[i].m_level) > 0) {
        m_log_priority = static_cast<uint32_t>(_colors[i].m_level);
      } else {
        const char* end_color = (_num_colors == i + 1)
                                    ? _buffer + _buffer_size
                                    : _colors[i + 1].m_position;
        size_t len = end_color - _colors[i].m_position;
        __android_log_print(s_log_levels[static_cast<uint32_t>(m_log_priority)],
            wn::utilities::gAndroidLogTag, "%.*s", static_cast<int32_t>(len),
            _colors[i].m_position);
      }
    }
  } else {
    __android_log_print(s_log_levels[static_cast<uint32_t>(m_log_priority)],
        wn::utilities::gAndroidLogTag, "%.*s", static_cast<int>(_buffer_size),
        _buffer);
  }
}

}  // namespace logging
}  // namespace wn

#endif  //__WN_CONSOLE_LOGGER_LINUX_INL__
<|endoftext|>


<commit_before>//===- MacroFusion.cpp - Macro Fusion -------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
/// \file This file contains the implementation of the DAG scheduling mutation
/// to pair instructions back to back.
//
//===----------------------------------------------------------------------===//

#include "llvm/CodeGen/MacroFusion.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/ScheduleDAGMutation.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

#define DEBUG_TYPE "machine-scheduler"

STATISTIC(NumFused, "Number of instr pairs fused");

using namespace llvm;

static cl::opt<bool> EnableMacroFusion("misched-fusion", cl::Hidden,
  cl::desc("Enable scheduling for macro fusion."), cl::init(true));

static void fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU,
                                SUnit &SecondSU) {
  // Create a single weak edge between the adjacent instrs. The only effect is
  // to cause bottom-up scheduling to heavily prioritize the clustered instrs.
  DAG.addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster));

  // Adjust the latency between the anchor instr and its
  // predecessors.
  for (SDep &IDep : SecondSU.Preds)
    if (IDep.getSUnit() == &FirstSU)
      IDep.setLatency(0);

  // Adjust the latency between the dependent instr and its
  // predecessors.
  for (SDep &IDep : FirstSU.Succs)
    if (IDep.getSUnit() == &SecondSU)
      IDep.setLatency(0);

  DEBUG(dbgs() << DAG.MF.getName() << "(): Macro fuse ";
        FirstSU.print(dbgs(), &DAG); dbgs() << " - ";
        SecondSU.print(dbgs(), &DAG); dbgs() << " /  ";
        dbgs() << DAG.TII->getName(FirstSU.getInstr()->getOpcode()) << " - " <<
                  DAG.TII->getName(SecondSU.getInstr()->getOpcode()) << '\n'; );

  if (&SecondSU != &DAG.ExitSU)
    // Make instructions dependent on FirstSU also dependent on SecondSU to
    // prevent them from being scheduled between FirstSU and and SecondSU.
    for (const SDep &SI : FirstSU.Succs) {
      if (SI.getSUnit() == &SecondSU)
        continue;
      DEBUG(dbgs() << "  Copy Succ ";
            SI.getSUnit()->print(dbgs(), &DAG); dbgs() << '\n';);
      DAG.addEdge(SI.getSUnit(), SDep(&SecondSU, SDep::Artificial));
    }

  ++NumFused;
}

namespace {

/// \brief Post-process the DAG to create cluster edges between instrs that may
/// be fused by the processor into a single operation.
class MacroFusion : public ScheduleDAGMutation {
  ShouldSchedulePredTy shouldScheduleAdjacent;
  bool FuseBlock;
  bool scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU);

public:
  MacroFusion(ShouldSchedulePredTy shouldScheduleAdjacent, bool FuseBlock)
    : shouldScheduleAdjacent(shouldScheduleAdjacent), FuseBlock(FuseBlock) {}

  void apply(ScheduleDAGInstrs *DAGInstrs) override;
};

} // end anonymous namespace

void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) {
  ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);

  if (FuseBlock)
    // For each of the SUnits in the scheduling block, try to fuse the instr in
    // it with one in its predecessors.
    for (SUnit &ISU : DAG->SUnits)
        scheduleAdjacentImpl(*DAG, ISU);

  if (DAG->ExitSU.getInstr())
    // Try to fuse the instr in the ExitSU with one in its predecessors.
    scheduleAdjacentImpl(*DAG, DAG->ExitSU);
}

/// \brief Implement the fusion of instr pairs in the scheduling DAG,
/// anchored at the instr in AnchorSU..
bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU) {
  const MachineInstr &AnchorMI = *AnchorSU.getInstr();
  const TargetInstrInfo &TII = *DAG.TII;
  const TargetSubtargetInfo &ST = DAG.MF.getSubtarget();

  // Check if the anchor instr may be fused.
  if (!shouldScheduleAdjacent(TII, ST, nullptr, AnchorMI))
    return false;

  // Explorer for fusion candidates among the dependencies of the anchor instr.
  for (SDep &Dep : AnchorSU.Preds) {
    // Ignore dependencies that don't enforce ordering.
    if (Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output ||
        Dep.isWeak())
      continue;

    SUnit &DepSU = *Dep.getSUnit();
    if (DepSU.isBoundaryNode())
      continue;

    const MachineInstr *DepMI = DepSU.getInstr();
    if (!shouldScheduleAdjacent(TII, ST, DepMI, AnchorMI))
      continue;

    fuseInstructionPair(DAG, DepSU, AnchorSU);
    return true;
  }

  return false;
}

std::unique_ptr<ScheduleDAGMutation>
llvm::createMacroFusionDAGMutation(
     ShouldSchedulePredTy shouldScheduleAdjacent) {
  if(EnableMacroFusion)
    return llvm::make_unique<MacroFusion>(shouldScheduleAdjacent, true);
  return nullptr;
}

std::unique_ptr<ScheduleDAGMutation>
llvm::createBranchMacroFusionDAGMutation(
     ShouldSchedulePredTy shouldScheduleAdjacent) {
  if(EnableMacroFusion)
    return llvm::make_unique<MacroFusion>(shouldScheduleAdjacent, false);
  return nullptr;
}
<commit_msg>[CodeGen] Improve the consistency of instruction fusion*<commit_after>//===- MacroFusion.cpp - Macro Fusion -------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
/// \file This file contains the implementation of the DAG scheduling mutation
/// to pair instructions back to back.
//
//===----------------------------------------------------------------------===//

#include "llvm/CodeGen/MacroFusion.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/ScheduleDAGMutation.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

#define DEBUG_TYPE "machine-scheduler"

STATISTIC(NumFused, "Number of instr pairs fused");

using namespace llvm;

static cl::opt<bool> EnableMacroFusion("misched-fusion", cl::Hidden,
  cl::desc("Enable scheduling for macro fusion."), cl::init(true));

static bool isHazard(const SDep &Dep) {
  return Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output;
}

static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU,
                                SUnit &SecondSU) {
  // Check that neither instr is already paired with another along the edge
  // between them.
  for (SDep &SI : FirstSU.Succs)
    if (SI.isCluster())
      return false;

  for (SDep &SI : SecondSU.Preds)
    if (SI.isCluster())
      return false;
  // Though the reachability checks above could be made more generic,
  // perhaps as part of ScheduleDAGMI::addEdge(), since such edges are valid,
  // the extra computation cost makes it less interesting in general cases.

  // Create a single weak edge between the adjacent instrs. The only effect is
  // to cause bottom-up scheduling to heavily prioritize the clustered instrs.
  if (!DAG.addEdge(&SecondSU, SDep(&FirstSU, SDep::Cluster)))
    return false;

  // Adjust the latency between both instrs.
  for (SDep &SI : FirstSU.Succs)
    if (SI.getSUnit() == &SecondSU)
      SI.setLatency(0);

  for (SDep &SI : SecondSU.Preds)
    if (SI.getSUnit() == &FirstSU)
      SI.setLatency(0);

  DEBUG(dbgs() << "Macro fuse: ";
        FirstSU.print(dbgs(), &DAG); dbgs() << " - ";
        SecondSU.print(dbgs(), &DAG); dbgs() << " /  ";
        dbgs() << DAG.TII->getName(FirstSU.getInstr()->getOpcode()) << " - " <<
                  DAG.TII->getName(SecondSU.getInstr()->getOpcode()) << '\n'; );

  // Make data dependencies from the FirstSU also dependent on the SecondSU to
  // prevent them from being scheduled between the FirstSU and the SecondSU.
  if (&SecondSU != &DAG.ExitSU)
    for (const SDep &SI : FirstSU.Succs) {
      SUnit *SU = SI.getSUnit();
      if (SI.isWeak() || isHazard(SI) ||
          SU == &DAG.ExitSU || SU == &SecondSU || SU->isPred(&SecondSU))
        continue;
      DEBUG(dbgs() << "  Bind ";
            SecondSU.print(dbgs(), &DAG); dbgs() << " - ";
            SU->print(dbgs(), &DAG); dbgs() << '\n';);
      DAG.addEdge(SU, SDep(&SecondSU, SDep::Artificial));
    }

  // Make the FirstSU also dependent on the dependencies of the SecondSU to
  // prevent them from being scheduled between the FirstSU and the SecondSU.
  if (&FirstSU != &DAG.EntrySU)
    for (const SDep &SI : SecondSU.Preds) {
      SUnit *SU = SI.getSUnit();
      if (SI.isWeak() || isHazard(SI) || &FirstSU == SU || FirstSU.isSucc(SU))
        continue;
      DEBUG(dbgs() << "  Bind ";
            SU->print(dbgs(), &DAG); dbgs() << " - ";
            FirstSU.print(dbgs(), &DAG); dbgs() << '\n';);
      DAG.addEdge(&FirstSU, SDep(SU, SDep::Artificial));
    }

  ++NumFused;
  return true;
}

namespace {

/// \brief Post-process the DAG to create cluster edges between instrs that may
/// be fused by the processor into a single operation.
class MacroFusion : public ScheduleDAGMutation {
  ShouldSchedulePredTy shouldScheduleAdjacent;
  bool FuseBlock;
  bool scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU);

public:
  MacroFusion(ShouldSchedulePredTy shouldScheduleAdjacent, bool FuseBlock)
    : shouldScheduleAdjacent(shouldScheduleAdjacent), FuseBlock(FuseBlock) {}

  void apply(ScheduleDAGInstrs *DAGInstrs) override;
};

} // end anonymous namespace

void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) {
  ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);

  if (FuseBlock)
    // For each of the SUnits in the scheduling block, try to fuse the instr in
    // it with one in its predecessors.
    for (SUnit &ISU : DAG->SUnits)
        scheduleAdjacentImpl(*DAG, ISU);

  if (DAG->ExitSU.getInstr())
    // Try to fuse the instr in the ExitSU with one in its predecessors.
    scheduleAdjacentImpl(*DAG, DAG->ExitSU);
}

/// \brief Implement the fusion of instr pairs in the scheduling DAG,
/// anchored at the instr in AnchorSU..
bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU) {
  const MachineInstr &AnchorMI = *AnchorSU.getInstr();
  const TargetInstrInfo &TII = *DAG.TII;
  const TargetSubtargetInfo &ST = DAG.MF.getSubtarget();

  // Check if the anchor instr may be fused.
  if (!shouldScheduleAdjacent(TII, ST, nullptr, AnchorMI))
    return false;

  // Explorer for fusion candidates among the dependencies of the anchor instr.
  for (SDep &Dep : AnchorSU.Preds) {
    // Ignore dependencies other than data or strong ordering.
    if (Dep.isWeak() || isHazard(Dep))
      continue;

    SUnit &DepSU = *Dep.getSUnit();
    if (DepSU.isBoundaryNode())
      continue;

    const MachineInstr *DepMI = DepSU.getInstr();
    if (!shouldScheduleAdjacent(TII, ST, DepMI, AnchorMI))
      continue;

    if (fuseInstructionPair(DAG, DepSU, AnchorSU))
      return true;
  }

  return false;
}

std::unique_ptr<ScheduleDAGMutation>
llvm::createMacroFusionDAGMutation(
     ShouldSchedulePredTy shouldScheduleAdjacent) {
  if(EnableMacroFusion)
    return llvm::make_unique<MacroFusion>(shouldScheduleAdjacent, true);
  return nullptr;
}

std::unique_ptr<ScheduleDAGMutation>
llvm::createBranchMacroFusionDAGMutation(
     ShouldSchedulePredTy shouldScheduleAdjacent) {
  if(EnableMacroFusion)
    return llvm::make_unique<MacroFusion>(shouldScheduleAdjacent, false);
  return nullptr;
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
 *
 * This file is part of Orfeo Toolbox
 *
 *     https://www.orfeo-toolbox.org/
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "otbWrapperCompositeApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbOGRDataToSamplePositionFilter.h"

namespace otb
{
namespace Wrapper
{

class TrainImagesRegression : public CompositeApplication
{
public:
  /** Standard class typedefs. */
  typedef TrainImagesRegression Self;
  typedef CompositeApplication Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  /** Standard macro */
  itkNewMacro(Self);

  /** Standard macro */
  itkTypeMacro( TrainImagesRegression, Superclass );

private:

  void DoInit() override
  {
    SetName("TrainImagesRegression");
    SetDescription("Train a regression model from multiple triplets of feature images, predictor images "
                    "and training vector data.");

    SetDocLongDescription(
        "Train a classifier from multiple triplets of feature images, predictor images and training vector data. \n\n"

        "The training vector data must contain polygons corresponding to the input sampling positions. This data "
        "is used to extract samples using pixel values in each band of the feature image and the corresponding "
        "ground truth extracted from the predictor image.\n\n".

        "At the end of the application, the mean square error between groundtruth and predicted values is computed using "
        "the output model and the validation vector data. Note that if no validation data is given, the training data "
        "will be used for validation.\n\n"

        "The number of training and validation samples can be specified with parameters. If no size is given, all samples will "
        "be used. \n\n"

        "This application is based on LibSVM, OpenCV Machine Learning, and Shark ML. "
        "The output of this application is a text model file, whose format corresponds to the "
        "ML model type chosen. There is no image nor vector data output.");
    SetDocLimitations("None");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso("TrainVectorRegression \n"
                  "TrainImagesClassifier");

    AddDocTag(Tags::Learning);

    SetOfficialDocLink();

    ClearApplications();

    InitIO();
    InitSampling();
    InitLearning();

    AddParameter( ParameterType_Bool, "cleanup", "Temporary files cleaning" );
    SetParameterDescription( "cleanup",
                           "If activated, the application will try to clean all temporary files it created" );
    SetParameterInt( "cleanup", 1);
  }

protected:

  /** Holds sampling parameters specific to training or validation */
  struct SamplingParameters
  {
    std::vector<std::string> inputVectorList;
    unsigned int numberOfSamples = 0;
    std::string filePrefix = "";
  };

  /** Adds a class field to the input vectors, this is needed to perform sampling operations */
  void AddRegressionField(const std::vector<std::string> & inputFileNames,
                          const std::string & filePrefix)
  {
    auto setFieldAppli = GetInternalApplication("setfield");
    auto& outputFileNames =  m_FileHandler[ filePrefix+"inputWithClassField" ];

    setFieldAppli->SetParameterString("fn", m_ClassFieldName);
    setFieldAppli->SetParameterString("fv", "0");

    for (unsigned int i =0; i < inputFileNames.size(); i++)
    {
      outputFileNames.push_back(GetParameterString("io.out")+"_"+filePrefix+"Withfield"+ std::to_string(i) +".shp");
      setFieldAppli->SetParameterString("in", inputFileNames[i]);
      setFieldAppli->SetParameterString("out", outputFileNames[i]);

      // Call ExecuteAndWriteOutput because VectorDataSetField's ExecuteInternal() does not write vector data.
      setFieldAppli->ExecuteAndWriteOutput();
    }
  }

  /** Prepare and execute polygonClassStatistics on each input vector data file */
  void ComputePolygonStatistics(const std::string & filePrefix)
  {
    auto polygonClassAppli = GetInternalApplication("polystat");
    auto& input = m_FileHandler[ filePrefix+"inputWithClassField" ];
    auto& output = m_FileHandler[ filePrefix+"statsFiles" ];
    FloatVectorImageListType* inputImageList = GetParameterImageList( "io.il" );

    for (unsigned int i =0; i < input.size(); i++)
    {
      output.push_back(GetParameterString("io.out")+"_"+filePrefix+"PolygonStats"+ std::to_string(i) +".xml");

      polygonClassAppli->SetParameterInputImage( "in", inputImageList->GetNthElement(i) );
      polygonClassAppli->SetParameterString( "vec", input[i]);
      polygonClassAppli->SetParameterString( "out", output[i]);

      polygonClassAppli->UpdateParameters();
      polygonClassAppli->SetParameterString( "field", m_ClassFieldName);

      ExecuteInternal( "polystat" );
    }

  }

  /** Compute the sampling rates using the computed statistics */
  void ComputeSamplingRate(const std::string & filePrefix, unsigned int numberOfSamples)
  {
    auto samplingRateAppli = GetInternalApplication("rates");

    samplingRateAppli->SetParameterStringList( "il", m_FileHandler[ filePrefix+"statsFiles" ]);

    std::string outputFileName = GetParameterString("io.out")+"_"+filePrefix+"rates.csv";
    samplingRateAppli->SetParameterString("out", outputFileName);

    if (numberOfSamples)
    {
      samplingRateAppli->SetParameterString("strategy", "constant");

      // TODO why nb is a string in MultiImageSamplingRate + MultiImageSamplingRate seems to return nb-1 samples ...
      samplingRateAppli->SetParameterString("strategy.constant.nb", std::to_string(GetParameterInt("sample.nt")));
    }
    else
    {
      samplingRateAppli->SetParameterString("strategy", "all");
    }

    ExecuteInternal( "rates");

    auto& rateFiles = m_FileHandler[filePrefix+"rateFiles"];
    for (unsigned int i = 0; i< m_FileHandler[filePrefix+"statsFiles"].size(); i++)
    {
      rateFiles.push_back(GetParameterString("io.out")+"_"+filePrefix+"rates_"+std::to_string(i+1)+".csv");
    }

  }

  /** Configure and execute Sample Selection on each vector using the computed rates and statistic files. */
  void SelectSamples(const std::string & filePrefix)
  {
    auto sampleSelection = GetInternalApplication("select");

    FloatVectorImageListType* inputImageList = GetParameterImageList( "io.il" );
    auto& inputVectorFiles = m_FileHandler[ filePrefix+"inputWithClassField" ];
    auto& outputVectorFiles = m_FileHandler[ filePrefix+"samples" ];
    auto& rateFiles = m_FileHandler [filePrefix+"rateFiles"];
    auto& statFiles = m_FileHandler [filePrefix+"statsFiles"];

    for (unsigned int i =0; i < inputVectorFiles.size(); i++)
    {
      outputVectorFiles.push_back(GetParameterString("io.out")+"_"+filePrefix+"samples"+std::to_string(i)+".shp");
      sampleSelection->SetParameterInputImage("in", inputImageList->GetNthElement(i));
      sampleSelection->SetParameterString("vec", inputVectorFiles[i]);
      sampleSelection->SetParameterString("instats", statFiles[i]);
      sampleSelection->SetParameterString("strategy", "byclass");
      sampleSelection->SetParameterString("strategy.byclass.in", rateFiles[i]);
      sampleSelection->SetParameterString("out", outputVectorFiles[i]);

      sampleSelection->UpdateParameters();
      sampleSelection->SetParameterString("field", m_ClassFieldName);

      ExecuteInternal("select");
    }
  }

  /** Configure and execute sampleExtraction. The application is called twice by input vector
   * first values are extracted from the feature image and then the groundtruth is extracted
   * from the predictor image.*/
  void ExtractSamples(const std::string & filePrefix)
  {
    auto sampleExtraction = GetInternalApplication("extraction");

    FloatVectorImageListType* featureImageList = GetParameterImageList( "io.il" );
    FloatVectorImageListType* predictorImageList = GetParameterImageList("io.ip");
    auto& vectorFiles = m_FileHandler[ filePrefix+"samples" ];

    for (unsigned int i =0; i < vectorFiles.size(); i++)
    {
      sampleExtraction->SetParameterString("vec", vectorFiles[i]);
      sampleExtraction->UpdateParameters();
      sampleExtraction->SetParameterString("field", m_ClassFieldName);

      //First Extraction : Values from the feature image.
      sampleExtraction->SetParameterInputImage("in", featureImageList->GetNthElement(i));
      sampleExtraction->SetParameterString("outfield", "prefix");
      sampleExtraction->SetParameterString("outfield.prefix.name", m_FeaturePrefix);
      ExecuteInternal("extraction");

      //Second Extraction : groundtruth from the predictor image.
      sampleExtraction->SetParameterInputImage("in", predictorImageList->GetNthElement(i));
      sampleExtraction->SetParameterString("outfield", "list");
      sampleExtraction->SetParameterStringList("outfield.list.names", {m_PredictionFieldName});
      ExecuteInternal("extraction");
    }
  }

  /** Configure and execute TrainVectorClassifier. Note that many parameters of TrainVectorClassifier
   * are shared with the main application during initialization. */
  void TrainModel()
  {
    auto trainVectorRegression = GetInternalApplication("training");

    auto& trainSampleFileNameList = m_FileHandler["trainsamples"];
    std::vector<std::string> featureNames;
    for (unsigned int i = 0; i< GetParameterStringList("io.vd").size(); i++)
    {
      featureNames.push_back(m_FeaturePrefix+std::to_string(i));
    }

    trainVectorRegression->SetParameterStringList("io.vd", trainSampleFileNameList);
    trainVectorRegression->UpdateParameters();
    trainVectorRegression->SetParameterString("cfield", m_PredictionFieldName);
    trainVectorRegression->SetParameterStringList("feat", featureNames);

    if( IsParameterEnabled( "io.valid" ) && HasValue( "io.valid" ) )
    {
      trainVectorRegression->SetParameterStringList("valid.vd", m_FileHandler["validsamples"]);
    }

    ExecuteInternal("training");
  }

  /** Init input/output parameters */
  void InitIO()
  {
    AddParameter( ParameterType_Group, "io", "Input and output data" );
    SetParameterDescription( "io", "This group of parameters allows setting input and output data." );

    AddParameter( ParameterType_InputImageList, "io.il", "Input Image List" );
    SetParameterDescription( "io.il", "A list of input images." );
    MandatoryOn( "io.il" );

    AddParameter( ParameterType_InputImageList, "io.ip", "Input Predictor Image List" );
    SetParameterDescription( "io.ip", "A list of input images." );
    MandatoryOn( "io.ip" );

    AddParameter( ParameterType_InputVectorDataList, "io.vd", "Input Vector Data List" );
    SetParameterDescription( "io.vd", "A list of vector data to select the training samples." );
    MandatoryOn( "io.vd" );

    AddParameter( ParameterType_InputVectorDataList, "io.valid", "Validation Vector Data List" );
    SetParameterDescription( "io.valid", "A list of vector data to select the validation samples." );
    MandatoryOff( "io.valid" );
  }

  /** Init sampling parameters and applications */
  void InitSampling()
  {
    AddApplication( "VectorDataSetField", "setfield", "Set additional vector field");
    AddApplication( "PolygonClassStatistics", "polystat", "Polygon analysis" );
    AddApplication( "MultiImageSamplingRate", "rates", "Sampling rates" );

    AddParameter( ParameterType_Group, "sample", "Sampling parameters" );
    SetParameterDescription( "sample", "This group of parameters allows setting sampling parameters" );

    AddParameter( ParameterType_Int, "sample.nt", "Number of training samples" );
    SetParameterDescription( "sample.nt", "Number of training samples." );
    MandatoryOff( "sample.nt" );

    AddParameter( ParameterType_Int, "sample.nv", "Number of validation samples" );
    SetParameterDescription( "sample.nv", "Number of validation samples." );
    MandatoryOff( "sample.nv" );

    AddApplication( "SampleSelection", "select", "Sample selection" );
    AddApplication( "SampleExtraction", "extraction", "Sample extraction" );

  }

  /** Init Learning applications and share learning parameters */
  void InitLearning()
  {
    AddApplication( "TrainVectorRegression", "training", "Train vector regression");

    ShareParameter( "io.imstat", "training.io.stats" );
    ShareParameter( "io.out", "training.io.out" );

    ShareParameter( "classifier", "training.classifier" );
    ShareParameter( "rand", "training.rand" );

    ShareParameter( "io.mse", "training.io.mse" );
  }

private:

  void DoUpdateParameters() override
  {
  }

  /** remove all file in the fileHandler std::map and clear the fileHandler std::map.
   * For .shp file, this method also removes .shx, .dbf and .prj associated files.*/
  bool ClearFileHandler()
  {
    bool res = true;
    for (const auto & fileNameList : m_FileHandler)
    {
      for (const auto & filename : fileNameList.second)
      {
        res = true;
        if( itksys::SystemTools::FileExists( filename ) )
        {
          size_t posExt = filename.rfind( '.' );
          if( posExt != std::string::npos && filename.compare( posExt, std::string::npos, ".shp" ) == 0 )
          {
            std::string shxPath = filename.substr( 0, posExt ) + std::string( ".shx" );
            std::string dbfPath = filename.substr( 0, posExt ) + std::string( ".dbf" );
            std::string prjPath = filename.substr( 0, posExt ) + std::string( ".prj" );
            itksys::SystemTools::RemoveFile( shxPath );
            itksys::SystemTools::RemoveFile( dbfPath );
            itksys::SystemTools::RemoveFile( prjPath );
          }
        res = itksys::SystemTools::RemoveFile( filename );
        }
      }
    }
    m_FileHandler.clear();
    return res;
  }

  void PerformSampling(const SamplingParameters & params)
  {
    std::vector<std::string> vectorData = params.inputVectorList;
    std::string filePrefix = params.filePrefix;
    unsigned int numberOfSamples = params.numberOfSamples;

    AddRegressionField(vectorData, filePrefix);
    ComputePolygonStatistics(filePrefix);
    ComputeSamplingRate(filePrefix, numberOfSamples);
    SelectSamples(filePrefix);
    ExtractSamples(filePrefix);
  }

  void DoExecute() override
  {
    SamplingParameters trainParams;
    trainParams.inputVectorList = GetParameterStringList("io.vd");
    trainParams.filePrefix="train";
    if (HasValue("sample.nt"))
      trainParams.numberOfSamples = GetParameterInt("sample.nt");

    PerformSampling(trainParams);

    if( IsParameterEnabled( "io.valid" ) && HasValue( "io.valid" ) )
    {
      SamplingParameters validParams;
      validParams.inputVectorList = GetParameterStringList("io.valid");
      validParams.filePrefix="valid";
      if (HasValue("sample.nv"))
        validParams.numberOfSamples = GetParameterInt("sample.nv");

      PerformSampling(validParams);
    }

    otbAppLogINFO( "Sampling Done." );

    TrainModel();

    if (GetParameterInt("cleanup"))
    {
      otbAppLogINFO( "Cleaning temporary files ..." );
      ClearFileHandler();
    }
  }

  /** Field used for sampling operations. */
  std::string m_ClassFieldName = "regclass";

  /** field containg the extracted regression groundtruth */
  std::string m_PredictionFieldName = "prediction";

  /** field containing the extracted pixel values for regression */
  std::string m_FeaturePrefix = "value_";

  /** Container containing the list of temporary files created during the execution*/
  std::map< std::string, std::vector<std::string>> m_FileHandler;

};

} //end namespace Wrapper
} //end namespace otb

OTB_APPLICATION_EXPORT(otb::Wrapper::TrainImagesRegression)
<commit_msg>BUG: remove . character<commit_after>/*
 * Copyright (C) 2005-2019 Centre National d'Etudes Spatiales (CNES)
 *
 * This file is part of Orfeo Toolbox
 *
 *     https://www.orfeo-toolbox.org/
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "otbWrapperCompositeApplication.h"
#include "otbWrapperApplicationFactory.h"

#include "otbOGRDataToSamplePositionFilter.h"

namespace otb
{
namespace Wrapper
{

class TrainImagesRegression : public CompositeApplication
{
public:
  /** Standard class typedefs. */
  typedef TrainImagesRegression Self;
  typedef CompositeApplication Superclass;
  typedef itk::SmartPointer<Self> Pointer;
  typedef itk::SmartPointer<const Self> ConstPointer;

  /** Standard macro */
  itkNewMacro(Self);

  /** Standard macro */
  itkTypeMacro( TrainImagesRegression, Superclass );

private:

  void DoInit() override
  {
    SetName("TrainImagesRegression");
    SetDescription("Train a regression model from multiple triplets of feature images, predictor images "
                    "and training vector data.");

    SetDocLongDescription(
        "Train a classifier from multiple triplets of feature images, predictor images and training vector data. \n\n"

        "The training vector data must contain polygons corresponding to the input sampling positions. This data "
        "is used to extract samples using pixel values in each band of the feature image and the corresponding "
        "ground truth extracted from the predictor image.\n\n"

        "At the end of the application, the mean square error between groundtruth and predicted values is computed using "
        "the output model and the validation vector data. Note that if no validation data is given, the training data "
        "will be used for validation.\n\n"

        "The number of training and validation samples can be specified with parameters. If no size is given, all samples will "
        "be used. \n\n"

        "This application is based on LibSVM, OpenCV Machine Learning, and Shark ML. "
        "The output of this application is a text model file, whose format corresponds to the "
        "ML model type chosen. There is no image nor vector data output.");
    SetDocLimitations("None");
    SetDocAuthors("OTB-Team");
    SetDocSeeAlso("TrainVectorRegression \n"
                  "TrainImagesClassifier");

    AddDocTag(Tags::Learning);

    SetOfficialDocLink();

    ClearApplications();

    InitIO();
    InitSampling();
    InitLearning();

    AddParameter( ParameterType_Bool, "cleanup", "Temporary files cleaning" );
    SetParameterDescription( "cleanup",
                           "If activated, the application will try to clean all temporary files it created" );
    SetParameterInt( "cleanup", 1);
  }

protected:

  /** Holds sampling parameters specific to training or validation */
  struct SamplingParameters
  {
    std::vector<std::string> inputVectorList;
    unsigned int numberOfSamples = 0;
    std::string filePrefix = "";
  };

  /** Adds a class field to the input vectors, this is needed to perform sampling operations */
  void AddRegressionField(const std::vector<std::string> & inputFileNames,
                          const std::string & filePrefix)
  {
    auto setFieldAppli = GetInternalApplication("setfield");
    auto& outputFileNames =  m_FileHandler[ filePrefix+"inputWithClassField" ];

    setFieldAppli->SetParameterString("fn", m_ClassFieldName);
    setFieldAppli->SetParameterString("fv", "0");

    for (unsigned int i =0; i < inputFileNames.size(); i++)
    {
      outputFileNames.push_back(GetParameterString("io.out")+"_"+filePrefix+"Withfield"+ std::to_string(i) +".shp");
      setFieldAppli->SetParameterString("in", inputFileNames[i]);
      setFieldAppli->SetParameterString("out", outputFileNames[i]);

      // Call ExecuteAndWriteOutput because VectorDataSetField's ExecuteInternal() does not write vector data.
      setFieldAppli->ExecuteAndWriteOutput();
    }
  }

  /** Prepare and execute polygonClassStatistics on each input vector data file */
  void ComputePolygonStatistics(const std::string & filePrefix)
  {
    auto polygonClassAppli = GetInternalApplication("polystat");
    auto& input = m_FileHandler[ filePrefix+"inputWithClassField" ];
    auto& output = m_FileHandler[ filePrefix+"statsFiles" ];
    FloatVectorImageListType* inputImageList = GetParameterImageList( "io.il" );

    for (unsigned int i =0; i < input.size(); i++)
    {
      output.push_back(GetParameterString("io.out")+"_"+filePrefix+"PolygonStats"+ std::to_string(i) +".xml");

      polygonClassAppli->SetParameterInputImage( "in", inputImageList->GetNthElement(i) );
      polygonClassAppli->SetParameterString( "vec", input[i]);
      polygonClassAppli->SetParameterString( "out", output[i]);

      polygonClassAppli->UpdateParameters();
      polygonClassAppli->SetParameterString( "field", m_ClassFieldName);

      ExecuteInternal( "polystat" );
    }

  }

  /** Compute the sampling rates using the computed statistics */
  void ComputeSamplingRate(const std::string & filePrefix, unsigned int numberOfSamples)
  {
    auto samplingRateAppli = GetInternalApplication("rates");

    samplingRateAppli->SetParameterStringList( "il", m_FileHandler[ filePrefix+"statsFiles" ]);

    std::string outputFileName = GetParameterString("io.out")+"_"+filePrefix+"rates.csv";
    samplingRateAppli->SetParameterString("out", outputFileName);

    if (numberOfSamples)
    {
      samplingRateAppli->SetParameterString("strategy", "constant");

      // TODO why nb is a string in MultiImageSamplingRate + MultiImageSamplingRate seems to return nb-1 samples ...
      samplingRateAppli->SetParameterString("strategy.constant.nb", std::to_string(GetParameterInt("sample.nt")));
    }
    else
    {
      samplingRateAppli->SetParameterString("strategy", "all");
    }

    ExecuteInternal( "rates");

    auto& rateFiles = m_FileHandler[filePrefix+"rateFiles"];
    for (unsigned int i = 0; i< m_FileHandler[filePrefix+"statsFiles"].size(); i++)
    {
      rateFiles.push_back(GetParameterString("io.out")+"_"+filePrefix+"rates_"+std::to_string(i+1)+".csv");
    }

  }

  /** Configure and execute Sample Selection on each vector using the computed rates and statistic files. */
  void SelectSamples(const std::string & filePrefix)
  {
    auto sampleSelection = GetInternalApplication("select");

    FloatVectorImageListType* inputImageList = GetParameterImageList( "io.il" );
    auto& inputVectorFiles = m_FileHandler[ filePrefix+"inputWithClassField" ];
    auto& outputVectorFiles = m_FileHandler[ filePrefix+"samples" ];
    auto& rateFiles = m_FileHandler [filePrefix+"rateFiles"];
    auto& statFiles = m_FileHandler [filePrefix+"statsFiles"];

    for (unsigned int i =0; i < inputVectorFiles.size(); i++)
    {
      outputVectorFiles.push_back(GetParameterString("io.out")+"_"+filePrefix+"samples"+std::to_string(i)+".shp");
      sampleSelection->SetParameterInputImage("in", inputImageList->GetNthElement(i));
      sampleSelection->SetParameterString("vec", inputVectorFiles[i]);
      sampleSelection->SetParameterString("instats", statFiles[i]);
      sampleSelection->SetParameterString("strategy", "byclass");
      sampleSelection->SetParameterString("strategy.byclass.in", rateFiles[i]);
      sampleSelection->SetParameterString("out", outputVectorFiles[i]);

      sampleSelection->UpdateParameters();
      sampleSelection->SetParameterString("field", m_ClassFieldName);

      ExecuteInternal("select");
    }
  }

  /** Configure and execute sampleExtraction. The application is called twice by input vector
   * first values are extracted from the feature image and then the groundtruth is extracted
   * from the predictor image.*/
  void ExtractSamples(const std::string & filePrefix)
  {
    auto sampleExtraction = GetInternalApplication("extraction");

    FloatVectorImageListType* featureImageList = GetParameterImageList( "io.il" );
    FloatVectorImageListType* predictorImageList = GetParameterImageList("io.ip");
    auto& vectorFiles = m_FileHandler[ filePrefix+"samples" ];

    for (unsigned int i =0; i < vectorFiles.size(); i++)
    {
      sampleExtraction->SetParameterString("vec", vectorFiles[i]);
      sampleExtraction->UpdateParameters();
      sampleExtraction->SetParameterString("field", m_ClassFieldName);

      //First Extraction : Values from the feature image.
      sampleExtraction->SetParameterInputImage("in", featureImageList->GetNthElement(i));
      sampleExtraction->SetParameterString("outfield", "prefix");
      sampleExtraction->SetParameterString("outfield.prefix.name", m_FeaturePrefix);
      ExecuteInternal("extraction");

      //Second Extraction : groundtruth from the predictor image.
      sampleExtraction->SetParameterInputImage("in", predictorImageList->GetNthElement(i));
      sampleExtraction->SetParameterString("outfield", "list");
      sampleExtraction->SetParameterStringList("outfield.list.names", {m_PredictionFieldName});
      ExecuteInternal("extraction");
    }
  }

  /** Configure and execute TrainVectorClassifier. Note that many parameters of TrainVectorClassifier
   * are shared with the main application during initialization. */
  void TrainModel()
  {
    auto trainVectorRegression = GetInternalApplication("training");

    auto& trainSampleFileNameList = m_FileHandler["trainsamples"];
    std::vector<std::string> featureNames;
    for (unsigned int i = 0; i< GetParameterStringList("io.vd").size(); i++)
    {
      featureNames.push_back(m_FeaturePrefix+std::to_string(i));
    }

    trainVectorRegression->SetParameterStringList("io.vd", trainSampleFileNameList);
    trainVectorRegression->UpdateParameters();
    trainVectorRegression->SetParameterString("cfield", m_PredictionFieldName);
    trainVectorRegression->SetParameterStringList("feat", featureNames);

    if( IsParameterEnabled( "io.valid" ) && HasValue( "io.valid" ) )
    {
      trainVectorRegression->SetParameterStringList("valid.vd", m_FileHandler["validsamples"]);
    }

    ExecuteInternal("training");
  }

  /** Init input/output parameters */
  void InitIO()
  {
    AddParameter( ParameterType_Group, "io", "Input and output data" );
    SetParameterDescription( "io", "This group of parameters allows setting input and output data." );

    AddParameter( ParameterType_InputImageList, "io.il", "Input Image List" );
    SetParameterDescription( "io.il", "A list of input images." );
    MandatoryOn( "io.il" );

    AddParameter( ParameterType_InputImageList, "io.ip", "Input Predictor Image List" );
    SetParameterDescription( "io.ip", "A list of input images." );
    MandatoryOn( "io.ip" );

    AddParameter( ParameterType_InputVectorDataList, "io.vd", "Input Vector Data List" );
    SetParameterDescription( "io.vd", "A list of vector data to select the training samples." );
    MandatoryOn( "io.vd" );

    AddParameter( ParameterType_InputVectorDataList, "io.valid", "Validation Vector Data List" );
    SetParameterDescription( "io.valid", "A list of vector data to select the validation samples." );
    MandatoryOff( "io.valid" );
  }

  /** Init sampling parameters and applications */
  void InitSampling()
  {
    AddApplication( "VectorDataSetField", "setfield", "Set additional vector field");
    AddApplication( "PolygonClassStatistics", "polystat", "Polygon analysis" );
    AddApplication( "MultiImageSamplingRate", "rates", "Sampling rates" );

    AddParameter( ParameterType_Group, "sample", "Sampling parameters" );
    SetParameterDescription( "sample", "This group of parameters allows setting sampling parameters" );

    AddParameter( ParameterType_Int, "sample.nt", "Number of training samples" );
    SetParameterDescription( "sample.nt", "Number of training samples." );
    MandatoryOff( "sample.nt" );

    AddParameter( ParameterType_Int, "sample.nv", "Number of validation samples" );
    SetParameterDescription( "sample.nv", "Number of validation samples." );
    MandatoryOff( "sample.nv" );

    AddApplication( "SampleSelection", "select", "Sample selection" );
    AddApplication( "SampleExtraction", "extraction", "Sample extraction" );

  }

  /** Init Learning applications and share learning parameters */
  void InitLearning()
  {
    AddApplication( "TrainVectorRegression", "training", "Train vector regression");

    ShareParameter( "io.imstat", "training.io.stats" );
    ShareParameter( "io.out", "training.io.out" );

    ShareParameter( "classifier", "training.classifier" );
    ShareParameter( "rand", "training.rand" );

    ShareParameter( "io.mse", "training.io.mse" );
  }

private:

  void DoUpdateParameters() override
  {
  }

  /** remove all file in the fileHandler std::map and clear the fileHandler std::map.
   * For .shp file, this method also removes .shx, .dbf and .prj associated files.*/
  bool ClearFileHandler()
  {
    bool res = true;
    for (const auto & fileNameList : m_FileHandler)
    {
      for (const auto & filename : fileNameList.second)
      {
        res = true;
        if( itksys::SystemTools::FileExists( filename ) )
        {
          size_t posExt = filename.rfind( '.' );
          if( posExt != std::string::npos && filename.compare( posExt, std::string::npos, ".shp" ) == 0 )
          {
            std::string shxPath = filename.substr( 0, posExt ) + std::string( ".shx" );
            std::string dbfPath = filename.substr( 0, posExt ) + std::string( ".dbf" );
            std::string prjPath = filename.substr( 0, posExt ) + std::string( ".prj" );
            itksys::SystemTools::RemoveFile( shxPath );
            itksys::SystemTools::RemoveFile( dbfPath );
            itksys::SystemTools::RemoveFile( prjPath );
          }
        res = itksys::SystemTools::RemoveFile( filename );
        }
      }
    }
    m_FileHandler.clear();
    return res;
  }

  void PerformSampling(const SamplingParameters & params)
  {
    std::vector<std::string> vectorData = params.inputVectorList;
    std::string filePrefix = params.filePrefix;
    unsigned int numberOfSamples = params.numberOfSamples;

    AddRegressionField(vectorData, filePrefix);
    ComputePolygonStatistics(filePrefix);
    ComputeSamplingRate(filePrefix, numberOfSamples);
    SelectSamples(filePrefix);
    ExtractSamples(filePrefix);
  }

  void DoExecute() override
  {
    SamplingParameters trainParams;
    trainParams.inputVectorList = GetParameterStringList("io.vd");
    trainParams.filePrefix="train";
    if (HasValue("sample.nt"))
      trainParams.numberOfSamples = GetParameterInt("sample.nt");

    PerformSampling(trainParams);

    if( IsParameterEnabled( "io.valid" ) && HasValue( "io.valid" ) )
    {
      SamplingParameters validParams;
      validParams.inputVectorList = GetParameterStringList("io.valid");
      validParams.filePrefix="valid";
      if (HasValue("sample.nv"))
        validParams.numberOfSamples = GetParameterInt("sample.nv");

      PerformSampling(validParams);
    }

    otbAppLogINFO( "Sampling Done." );

    TrainModel();

    if (GetParameterInt("cleanup"))
    {
      otbAppLogINFO( "Cleaning temporary files ..." );
      ClearFileHandler();
    }
  }

  /** Field used for sampling operations. */
  std::string m_ClassFieldName = "regclass";

  /** field containg the extracted regression groundtruth */
  std::string m_PredictionFieldName = "prediction";

  /** field containing the extracted pixel values for regression */
  std::string m_FeaturePrefix = "value_";

  /** Container containing the list of temporary files created during the execution*/
  std::map< std::string, std::vector<std::string>> m_FileHandler;

};

} //end namespace Wrapper
} //end namespace otb

OTB_APPLICATION_EXPORT(otb::Wrapper::TrainImagesRegression)
<|endoftext|>


<commit_before>/* symtable.cc
 * Hash table representing the symbol table.
 * the symbol table.
 * UIdaho CS445 120++ Compiler
 * author: Chris Waltrip <walt2178@vandals.uidaho.edu>
 */
#include <iostream>
#include <sstream>
#include <string>

#include "symtable.hh"
#include "exception.hh"

/*
AbstractSymbol::AbstractSymbol(std::string n, TypenameEntry t)
			: name(n), type(t) { }
*/
AbstractSymbol::AbstractSymbol(std::string n, std::string t)
			: name(n), type(t) { }
std::string AbstractSymbol::to_string(std::size_t depth) {
	std::string spaces = std::string(depth, '>');
	return(spaces + this->type + " " + this->name);
}

/*
 * The only time that SymbolTable's parent will be NULL is the Global Symbol
 * Table.  Given this and the n-ary tree (represented using std::deque),
 * resolving scope should be relatively simple.
 */
SymbolTable::SymbolTable(SymbolTable *p) {
	this->parent = p;
}

/*
 * Just a wrapper function for nesting scopes.
 */
void SymbolTable::add_sub_table(SymbolTable *k) {
	this->kids.push_back(k);
}
/*
 * This hashing method is identical to the Typename Table hash method.
 */
std::size_t SymbolTable::hash(std::string name) {
	std::size_t hash = 0;
	const char* s = name.c_str();
	while(*s) {
		hash = hash * 101 + *s++;
	}
	return hash % this->HASHTABLE_SIZE;
}

bool SymbolTable::insert(std::string n, AbstractSymbol *s) {
	if(!this->symbol_exists(n)) {
		std::size_t h = this->hash(n);
		this->bucket[h].push_back(s);
		return true;
	}
	throw EDuplicateSymbol();
	return false; /* Remove after refactor */
}

bool SymbolTable::empty() {
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		if(!this->bucket[i].empty())
			return false;
	}
	return true;
}

AbstractSymbol* SymbolTable::get_symbol(std::string name) {
	std::size_t h = this->hash(name);
	std::deque<AbstractSymbol*> b = this->bucket[h];
	std::deque<AbstractSymbol*>::iterator it;
	for(it = b.begin(); it != b.end(); ++it) {
		std::size_t i = it - b.begin();
		if(*(b[i]) == name) {
			return b[i];
		}
	}
	throw ENoSymbolEntry();
}

/*
 * Tries to get the symbol from the current scope.  If get_symbol throws an
 * ENoSymbolEntry, then the current scope doesn't have the symbol that is
 * being search for.  If the parent symbol table is NULL then we have reached
 * the global symbol table and the symbol doesn't exist so throw
 * ENoSymbolEntry again and let the calling function handle the exception. If
 * the symbol is found, it's returned.
 */
AbstractSymbol* SymbolTable::get_scoped_symbol(std::string name) {
	AbstractSymbol *symb;
	try {
		symb = this->get_symbol(name);
		return symb;
	} catch(ENoSymbolEntry e) {
		if(this->parent == NULL) {
			throw ENoSymbolEntry();
		} else {
			/* Will never be NULL because exception is thrown */
			return this->parent->get_scoped_symbol(name);
		}
	}
}

/*
 * Will return true if a symbol is found.  If get_symbol throws
 * an ENoSymbolEntry exception, then the symbol isn't in the symbol
 * table.  This only checks for the existance in the local symbol table.
 */
bool SymbolTable::symbol_exists(std::string name) {
	try {
		this->get_symbol(name);
	} catch(ENoSymbolEntry e) {
		return false;
	}
	return true;
}

void SymbolTable::print_table(std::size_t depth) {
	std::clog << this->to_string(depth) << std::endl;
}

std::string SymbolTable::to_string(std::size_t depth) {
	std::stringstream ss;
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		std::deque<AbstractSymbol*> b = this->bucket[i];
		std::deque<AbstractSymbol*>::iterator it;
		for(it = b.begin(); it != b.end(); ++it) {
			std::size_t index = it - b.begin();
			ss << (b[index])->to_string(depth) << std::endl;
			//ss << (*it)->to_string(depth) << std::endl;
		}
	}
	return ss.str();
}

/* Basic Symbol constructor */
BasicSymbol::BasicSymbol(std::string n, std::string t, bool p)
				: AbstractSymbol(n,t), pointer(p) { }

/* Function symbol constructor */
FunctionSymbol::FunctionSymbol(std::string n, std::string t, bool p, bool d)
				: AbstractSymbol(n,t), pointer(p),
				def_needed(d) { }

std::string FunctionSymbol::to_string(std::size_t depth) {
	std::stringstream ss;
	ss << AbstractSymbol::to_string(depth) << std::endl;
	ss << (this->params)->to_string(depth+1) << std::endl;
	ss << this->(locals->to_string(depth+1)) << std::endl;
	return ss.str();
}<commit_msg>fixing function symbol table additions<commit_after>/* symtable.cc
 * Hash table representing the symbol table.
 * the symbol table.
 * UIdaho CS445 120++ Compiler
 * author: Chris Waltrip <walt2178@vandals.uidaho.edu>
 */
#include <iostream>
#include <sstream>
#include <string>

#include "symtable.hh"
#include "exception.hh"

/*
AbstractSymbol::AbstractSymbol(std::string n, TypenameEntry t)
			: name(n), type(t) { }
*/
AbstractSymbol::AbstractSymbol(std::string n, std::string t)
			: name(n), type(t) { }
std::string AbstractSymbol::to_string(std::size_t depth) {
	std::string spaces = std::string(depth, '>');
	return(spaces + this->type + " " + this->name);
}

/*
 * The only time that SymbolTable's parent will be NULL is the Global Symbol
 * Table.  Given this and the n-ary tree (represented using std::deque),
 * resolving scope should be relatively simple.
 */
SymbolTable::SymbolTable(SymbolTable *p) {
	this->parent = p;
}

/*
 * Just a wrapper function for nesting scopes.
 */
void SymbolTable::add_sub_table(SymbolTable *k) {
	this->kids.push_back(k);
}
/*
 * This hashing method is identical to the Typename Table hash method.
 */
std::size_t SymbolTable::hash(std::string name) {
	std::size_t hash = 0;
	const char* s = name.c_str();
	while(*s) {
		hash = hash * 101 + *s++;
	}
	return hash % this->HASHTABLE_SIZE;
}

bool SymbolTable::insert(std::string n, AbstractSymbol *s) {
	if(!this->symbol_exists(n)) {
		std::size_t h = this->hash(n);
		this->bucket[h].push_back(s);
		return true;
	}
	throw EDuplicateSymbol();
	return false; /* Remove after refactor */
}

bool SymbolTable::empty() {
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		if(!this->bucket[i].empty())
			return false;
	}
	return true;
}

AbstractSymbol* SymbolTable::get_symbol(std::string name) {
	std::size_t h = this->hash(name);
	std::deque<AbstractSymbol*> b = this->bucket[h];
	std::deque<AbstractSymbol*>::iterator it;
	for(it = b.begin(); it != b.end(); ++it) {
		std::size_t i = it - b.begin();
		if(*(b[i]) == name) {
			return b[i];
		}
	}
	throw ENoSymbolEntry();
}

/*
 * Tries to get the symbol from the current scope.  If get_symbol throws an
 * ENoSymbolEntry, then the current scope doesn't have the symbol that is
 * being search for.  If the parent symbol table is NULL then we have reached
 * the global symbol table and the symbol doesn't exist so throw
 * ENoSymbolEntry again and let the calling function handle the exception. If
 * the symbol is found, it's returned.
 */
AbstractSymbol* SymbolTable::get_scoped_symbol(std::string name) {
	AbstractSymbol *symb;
	try {
		symb = this->get_symbol(name);
		return symb;
	} catch(ENoSymbolEntry e) {
		if(this->parent == NULL) {
			throw ENoSymbolEntry();
		} else {
			/* Will never be NULL because exception is thrown */
			return this->parent->get_scoped_symbol(name);
		}
	}
}

/*
 * Will return true if a symbol is found.  If get_symbol throws
 * an ENoSymbolEntry exception, then the symbol isn't in the symbol
 * table.  This only checks for the existance in the local symbol table.
 */
bool SymbolTable::symbol_exists(std::string name) {
	try {
		this->get_symbol(name);
	} catch(ENoSymbolEntry e) {
		return false;
	}
	return true;
}

void SymbolTable::print_table(std::size_t depth) {
	std::clog << this->to_string(depth) << std::endl;
}

std::string SymbolTable::to_string(std::size_t depth) {
	std::stringstream ss;
	for(std::size_t i = 0; i < this->HASHTABLE_SIZE; ++i) {
		std::deque<AbstractSymbol*> b = this->bucket[i];
		std::deque<AbstractSymbol*>::iterator it;
		for(it = b.begin(); it != b.end(); ++it) {
			std::size_t index = it - b.begin();
			ss << (b[index])->to_string(depth) << std::endl;
			//ss << (*it)->to_string(depth) << std::endl;
		}
	}
	return ss.str();
}

/* Basic Symbol constructor */
BasicSymbol::BasicSymbol(std::string n, std::string t, bool p)
				: AbstractSymbol(n,t), pointer(p) { }

/* Function symbol constructor */
FunctionSymbol::FunctionSymbol(std::string n, std::string t, bool p, bool d)
				: AbstractSymbol(n,t), pointer(p),
				def_needed(d) { }

std::string FunctionSymbol::to_string(std::size_t depth) {
	std::stringstream ss;
	ss << AbstractSymbol::to_string(depth) << std::endl;
	ss << (this->params)->to_string(depth+1) << std::endl;
	ss << this->locals->to_string(depth+1) << std::endl;
	return ss.str();
}<|endoftext|>