<commit_before>#include "defs.hpp"
#include <Grappa.hpp>
#include <Collective.hpp>
#include <Delegate.hpp>
#include <ParallelLoop.hpp>
#include <GlobalCompletionEvent.hpp>
#include <GlobalHashSet.hpp>

using namespace Grappa;
namespace d = Grappa::delegate;

DECLARE_int64(cc_hash_size);
GRAPPA_DEFINE_STAT(SimpleStatistic<graphint>, cc_intermediate_set_size, 0);

static graph g;

static GlobalAddress<graphint> colors;
static GlobalAddress<graphint> visited;
static GlobalAddress<color_t> marks;
static bool changed;
static graphint ncomponents;

GlobalCompletionEvent gce;

struct Edge {
  graphint start, end;
  bool operator==(const Edge& e) const { return e.start == start && e.end == end; }
  friend std::ostream& operator<<(std::ostream& o, const Edge& e);
};
std::ostream& operator<<(std::ostream& o, const Edge& e) {
  o << "{" << e.start << "," << e.end << "}";
  return o;
}
namespace std {
  template <> struct hash<Edge> {
    typedef size_t result_type;
    typedef Edge argument_type;
    size_t operator()(Edge const & e) const noexcept {
      return static_cast<size_t>(e.start ^ e.end);
    }
  };
}

using EdgeHashSet = GlobalHashSet<Edge>;
static GlobalAddress<EdgeHashSet> component_edges;

template< GlobalCompletionEvent * GCE >
void explore(graphint v, graphint mycolor) {
  if (mycolor < 0) {
    auto claimed = d::call(colors+v, [v](graphint * c){
      if (*c < 0) {
        *c = v;
        return true;
      } else {
        return false;
      }
    });
    if (!claimed) return;
    mycolor = v;
  }

  // visit neighbors of v
  graphint _c[2]; Incoherent<graphint>::RO c(g.edgeStart+v, 2, _c);

  forall_localized_async<GCE>(g.endVertex+c[0], c[1]-c[0], [mycolor](graphint& ev){
    // TODO: make async
    GCE->enroll();
    Core origin = mycore();
    auto colors_ev = colors+ev;
    send_message(colors_ev.core(), [origin,mycolor,colors_ev]{
      auto ec = colors_ev.pointer();
      size_t ev = colors_ev - colors;
      if (*ec < 0) {
        *ec = mycolor;
        publicTask([ev,origin,mycolor]{
          explore<GCE>(ev,mycolor);
          complete(make_global(GCE,origin));
        });
      } else {
        // already had color
        // TODO: avoid spawning task? (i.e. make insert() async)
        privateTask([ec,mycolor,origin]{
          Edge edge = (*ec > mycolor) ? Edge{mycolor,*ec} : Edge{*ec,mycolor};
          component_edges->insert(edge);
          complete(make_global(GCE,origin));
        });
      }
    });
  });
}

/// Takes a graph as input and an array with length NV.  The array D will store
/// the coloring of each component.  The coloring will be using vertex IDs and
/// therefore will be an integer between 0 and NV-1.  The function returns the
/// total number of components.
graphint connectedComponents(graph * in_g) {
  const auto NV = in_g->numVertices;

  auto _component_edges = EdgeHashSet::create(FLAGS_cc_hash_size);

  auto _colors = global_alloc<graphint>(NV);
  auto _visited = global_alloc<graphint>(NV);
  auto _in_g = *in_g;
  call_on_all_cores([_colors,_visited,_in_g,_component_edges]{
    component_edges = _component_edges;
    colors = _colors;
    visited = _visited;
    g = _in_g;
  });

  forall_localized(colors, NV, [](int64_t v, graphint& c){ c = -v-1; });

  ///////////////////////////////////////////////////////////////
  // Find component edges
  forall_localized<&gce,256>(colors, NV, [](int64_t v, graphint& c){
    explore<&gce>(v,c);
  });

  cc_intermediate_set_size = component_edges->size();

  ///////////////////////////////////////////////////////////////
  //
  int pass = 0;
  do {
    VLOG(0) << "pass " << pass;
    call_on_all_cores([]{ changed = false; });

    // Hook
    VLOG(0) << "hook";
    component_edges->forall_keys([](Edge& e){
      graphint ci = d::read(colors+e.start),
               cj = d::read(colors+e.end);

      bool lchanged = false;

      if ( ci < cj ) {
        lchanged = d::call(colors+cj, [cj](graphint* ccj){
          if (*ccj == cj) { *ccj = cj; return true; }
          else { return false; }
        });
      }
      if (!lchanged && cj < ci) {
        lchanged = d::call(colors+ci, [ci](graphint* cci){
          if (*cci == ci) { *cci = ci; return true; }
          else { return false; }
        });
      }

      if (lchanged) { changed = true; }
    });

    // Compress
    VLOG(0) << "compress";
    component_edges->forall_keys([](Edge& e){
      auto compress = [](graphint i) {
        graphint ci, cci, nc;
        ci = nc = d::read(colors+i);
        while ( nc != (cci=d::read(colors+nc)) ) { nc = d::read(colors+cci); }
        if (nc != ci) {
          changed = true;
          d::write(colors+i, nc);
        }
      };

      compress(e.start);
      compress(e.end);
    });
    pass++;
    call_on_all_cores([]{ VLOG(0) << "changed = " << changed; });
  } while (reduce<bool,collective_or>(&changed) == true);

  auto propagate = [](graphint v) {

  };

  graphint ncomponents = component_edges->size();
  VLOG(0) << "component_edges.size = " << component_edges->size();
  component_edges->forall_keys([](Edge& e){ VLOG(0) << e; });

  return ncomponents;
}
<commit_msg>First attempt at propagating (doesn't work)<commit_after>#include "defs.hpp"
#include <Grappa.hpp>
#include <Collective.hpp>
#include <Delegate.hpp>
#include <ParallelLoop.hpp>
#include <GlobalCompletionEvent.hpp>
#include <GlobalHashSet.hpp>
#include <GlobalCounter.hpp>
#include <Array.hpp>

using namespace Grappa;
namespace d = Grappa::delegate;

DECLARE_int64(cc_hash_size);

static graph g;

static GlobalAddress<graphint> colors;
static GlobalAddress<graphint> visited;
static GlobalAddress<color_t> marks;
static bool changed;
static graphint ncomponents;

GlobalCompletionEvent gce;

struct Edge {
  graphint start, end;
  bool operator==(const Edge& e) const { return e.start == start && e.end == end; }
  friend std::ostream& operator<<(std::ostream& o, const Edge& e);
};
std::ostream& operator<<(std::ostream& o, const Edge& e) {
  o << "{" << e.start << "," << e.end << "}";
  return o;
}
namespace std {
  template <> struct hash<Edge> {
    typedef size_t result_type;
    typedef Edge argument_type;
    size_t operator()(Edge const & e) const noexcept {
      return static_cast<size_t>(e.start ^ e.end);
    }
  };
}

using EdgeHashSet = GlobalHashSet<Edge>;
static GlobalAddress<EdgeHashSet> component_edges;

template< GlobalCompletionEvent * GCE >
void explore(graphint v, graphint mycolor) {
  if (mycolor < 0) {
    auto claimed = d::call(colors+v, [v](graphint * c){
      if (*c < 0) {
        *c = v;
        return true;
      } else {
        return false;
      }
    });
    if (!claimed) return;
    mycolor = v;
  }

  // visit neighbors of v
  graphint _c[2]; Incoherent<graphint>::RO c(g.edgeStart+v, 2, _c);

  forall_localized_async<GCE>(g.endVertex+c[0], c[1]-c[0], [mycolor](graphint& ev){
    // TODO: make async
    GCE->enroll();
    Core origin = mycore();
    auto colors_ev = colors+ev;
    send_message(colors_ev.core(), [origin,mycolor,colors_ev]{
      auto ec = colors_ev.pointer();
      size_t ev = colors_ev - colors;
      if (*ec < 0) {
        *ec = mycolor;
        publicTask([ev,origin,mycolor]{
          explore<GCE>(ev,mycolor);
          complete(make_global(GCE,origin));
        });
      } else {
        // already had color
        // TODO: avoid spawning task? (i.e. make insert() async)
        privateTask([ec,mycolor,origin]{
          Edge edge = (*ec > mycolor) ? Edge{mycolor,*ec} : Edge{*ec,mycolor};
          component_edges->insert(edge);
          complete(make_global(GCE,origin));
        });
      }
    });
  });
}

graphint dfs(graphint root) {
  graphint mycolor = root;
  // VLOG(0) << "dfs(" << root << ")";
  std::function<bool(graphint)> rec = [&mycolor,&rec](graphint v) {
    bool found = false;
    graphint _c[2]; Incoherent<graphint>::RO c(g.edgeStart+v, 2, _c);
    // VLOG(0) << "rec(" << v << ")[" << c[0] << " - " << c[1] << "]";
    for (graphint i=c[0]; i<c[1]; i++) {
      graphint ev = d::read(g.endVertex+i);
      // VLOG(0) << "[" << i-c[0] << "]: " << ev;
      if (d::read(visited+ev)) {
        mycolor = d::read(colors+ev);
        // VLOG(0) << "  found -> " << mycolor;
        found = true;
        break;
      } else if (rec(ev)) {
        // VLOG(0) << "  found (" << mycolor << ")";
        found = true;
        break;
      }
    }
    return found;
  };

  // CHECK(rec(root));
  return mycolor;
}

/// Takes a graph as input and an array with length NV.  The array D will store
/// the coloring of each component.  The coloring will be using vertex IDs and
/// therefore will be an integer between 0 and NV-1.  The function returns the
/// total number of components.
graphint connectedComponents(graph * in_g) {
  const auto NV = in_g->numVertices;

  auto _component_edges = EdgeHashSet::create(FLAGS_cc_hash_size);

  auto _colors = global_alloc<graphint>(NV);
  auto _visited = global_alloc<graphint>(NV);
  auto _in_g = *in_g;
  call_on_all_cores([_colors,_visited,_in_g,_component_edges]{
    component_edges = _component_edges;
    colors = _colors;
    visited = _visited;
    g = _in_g;
  });

  forall_localized(colors, NV, [](int64_t v, graphint& c){ c = -v-1; });

  ///////////////////////////////////////////////////////////////
  // Find component edges
  forall_localized<&gce,256>(colors, NV, [](int64_t v, graphint& c){
    explore<&gce>(v,c);
  });

  LOG(INFO) << "cc_intermediate_set_size: " << component_edges->size();
  component_edges->forall_keys([](Edge& e){ VLOG(0) << e; });
  VLOG(0) << util::array_str("colors: ", colors, NV);

  ///////////////////////////////////////////////////////////////
  //
  int npass = 0;
  do {
    DVLOG(2) << "npass " << npass;
    call_on_all_cores([]{ changed = false; });

    // Hook
    DVLOG(2) << "hook";
    component_edges->forall_keys([NV](Edge& e){
      graphint i = e.start, j = e.end;
      CHECK_LT(i, NV);
      CHECK_LT(j, NV);
      graphint ci = d::read(colors+e.start),
               cj = d::read(colors+e.end);
      CHECK_LT(ci, NV);
      CHECK_LT(cj, NV);
      bool lchanged = false;

      if ( ci < cj ) {
        lchanged = lchanged || d::call(colors+cj, [ci,cj](graphint* ccj){
          if (*ccj == cj) { *ccj = ci; return true; }
          else { return false; }
        });
      }
      if (!lchanged && cj < ci) {
        lchanged = lchanged || d::call(colors+ci, [ci,cj](graphint* cci){
          if (*cci == ci) { *cci = cj; return true; }
          else { return false; }
        });
      }

      if (lchanged) { changed = true; }
    });

    // Compress
    DVLOG(2) << "compress";
    component_edges->forall_keys([NV](Edge& e){
      auto compress = [NV](graphint i) {
        graphint ci, cci, nc;
        ci = nc = d::read(colors+i);
        CHECK_LT(ci, NV);
        CHECK_LT(nc, NV);
        while ( nc != (cci=d::read(colors+nc)) ) { nc = d::read(colors+cci); CHECK_LT(nc, NV); }
        if (nc != ci) {
          changed = true;
          d::write(colors+i, nc);
        }
      };

      compress(e.start);
      compress(e.end);
    });
    npass++;
  } while (reduce<bool,collective_or>(&changed) == true);

  LOG(INFO) << "cc_pram_passes: " << npass;

  // auto search = [](graphint v) {
  //   graphint _c[2]; Incoherent<graphint>::RO c(g.edgeStart+v, 2, _c);
  //   forall_localized_async<GCE>(g.endVertex+c[0], c[1]-c[0], [mycolor](graphint& ev){
  //     if (d::fetch_and_add(visited+ev, 1) == 0) {
  //       d::write(colors+ev, mycolor);
  //       publicTask<&gce>([ev,mycolor]{ search(ev,mycolor); });
  //     }
  //   });
  // };

  Grappa::memset(visited, 0, NV);
  component_edges->forall_keys([](Edge& e){
    d::write_async(*shared_pool, visited+e.start, 1);
    d::write_async(*shared_pool, visited+e.end, 1);
  });
  forall_localized(visited, NV, [](int64_t v, graphint& valid){
    if (!valid) {
      graphint newcolor = dfs(v);
      d::write(colors+v, newcolor);
      d::write(visited+v, 1);
    }
  });

  VLOG(0) << util::array_str("colors: ", colors, NV);

  auto nca = GlobalCounter::create();
  forall_localized(colors, NV, [nca](int64_t v, graphint& c){ if (c == v) nca->incr(); });
  graphint ncomponents = nca->count();
  nca->destroy();
  VLOG(0) << "ncomponents: " << ncomponents;
  return ncomponents;
}
<|endoftext|>


<commit_before><commit_msg>change data_set path<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2015 GitHub, Inc.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

#include "atom/browser/atom_resource_dispatcher_host_delegate.h"

#include "atom/browser/atom_browser_context.h"
#include "atom/browser/login_handler.h"
#include "atom/browser/web_contents_permission_helper.h"
#include "atom/browser/web_contents_preferences.h"
#include "atom/common/atom_constants.h"
#include "atom/common/platform_util.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/download_manager.h"
#include "content/public/browser/stream_info.h"
#include "content/public/browser/render_frame_host.h"
#include "net/base/escape.h"
#include "net/ssl/client_cert_store.h"
#include "net/url_request/url_request.h"
#include "url/gurl.h"

#if defined(USE_NSS_CERTS)
#include "net/ssl/client_cert_store_nss.h"
#elif defined(OS_WIN)
#include "net/ssl/client_cert_store_win.h"
#elif defined(OS_MACOSX)
#include "net/ssl/client_cert_store_mac.h"
#endif

using content::BrowserThread;

namespace atom {

namespace {

void OnOpenExternal(const GURL& escaped_url,
                    bool allowed) {
  if (allowed)
    platform_util::OpenExternal(
#if defined(OS_WIN)
        base::UTF8ToUTF16(escaped_url.spec()),
#else
        escaped_url,
#endif
        true);
}

void HandleExternalProtocolInUI(
    const GURL& url,
    const content::ResourceRequestInfo::WebContentsGetter& web_contents_getter,
    bool has_user_gesture) {
  content::WebContents* web_contents = web_contents_getter.Run();
  if (!web_contents)
    return;

  auto permission_helper =
      WebContentsPermissionHelper::FromWebContents(web_contents);
  if (!permission_helper)
    return;

  GURL escaped_url(net::EscapeExternalHandlerValue(url.spec()));
  auto callback = base::Bind(&OnOpenExternal, escaped_url);
  permission_helper->RequestOpenExternalPermission(callback, has_user_gesture);
}

void OnPdfResourceIntercepted(
	const GURL& original_url,
	int frame_tree_node_id,
	const content::ResourceRequestInfo::WebContentsGetter&
	web_contents_getter) {
	content::WebContents* web_contents = web_contents_getter.Run();
	if (!web_contents)
		return;

	if (!WebContentsPreferences::IsPluginsEnabled(web_contents)) {
		auto browser_context = web_contents->GetBrowserContext();
		auto download_manager =
			content::BrowserContext::GetDownloadManager(browser_context);

		download_manager->DownloadUrl(
			content::DownloadUrlParameters::CreateForWebContentsMainFrame(
				web_contents, original_url));
		return;
	}

	// The URL passes the original pdf resource url, that will be requested
	// by the webui page.
	// chrome://pdf-viewer/index.html?src=https://somepage/123.pdf
	content::NavigationController::LoadURLParams params(GURL(base::StringPrintf(
		"%sindex.html?%s=%s", kPdfViewerUIOrigin, kPdfPluginSrc,
		net::EscapeUrlEncodedData(original_url.spec(), false).c_str())));
	params.frame_tree_node_id = frame_tree_node_id;
	web_contents->GetController().LoadURLWithParams(params);
}

}  // namespace

AtomResourceDispatcherHostDelegate::AtomResourceDispatcherHostDelegate() {
}

bool AtomResourceDispatcherHostDelegate::HandleExternalProtocol(
    const GURL& url,
    content::ResourceRequestInfo* info) {
  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
                          base::Bind(&HandleExternalProtocolInUI,
                                     url,
                                     info->GetWebContentsGetterForRequest(),
                                     info->HasUserGesture()));
  return true;
}

content::ResourceDispatcherHostLoginDelegate*
AtomResourceDispatcherHostDelegate::CreateLoginDelegate(
    net::AuthChallengeInfo* auth_info,
    net::URLRequest* request) {
  return new LoginHandler(auth_info, request);
}

std::unique_ptr<net::ClientCertStore>
AtomResourceDispatcherHostDelegate::CreateClientCertStore(
    content::ResourceContext* resource_context) {
  #if defined(USE_NSS_CERTS)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreNSS(
        net::ClientCertStoreNSS::PasswordDelegateFactory()));
  #elif defined(OS_WIN)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreWin());
  #elif defined(OS_MACOSX)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreMac());
  #elif defined(USE_OPENSSL)
    return std::unique_ptr<net::ClientCertStore>();
  #endif
}

bool AtomResourceDispatcherHostDelegate::ShouldInterceptResourceAsStream(
	net::URLRequest* request,
	const base::FilePath& plugin_path,
	const std::string& mime_type,
	GURL* origin,
	std::string* payload) {
	const content::ResourceRequestInfo* info =
		content::ResourceRequestInfo::ForRequest(request);

	int render_process_host_id;
	int render_frame_id;
	info->GetAssociatedRenderFrame(&render_process_host_id, &render_frame_id);
	content::RenderFrameHost* rfh =
		content::RenderFrameHost::FromID(render_process_host_id, render_frame_id);

	int frame_tree_node_id = rfh->GetFrameTreeNodeId();

	if (mime_type == "application/pdf") {
		*origin = GURL(kPdfViewerUIOrigin);
		content::BrowserThread::PostTask(
			BrowserThread::UI, FROM_HERE,
			base::Bind(&OnPdfResourceIntercepted, request->url(),
				frame_tree_node_id, info->GetWebContentsGetterForRequest()));
		return true;
	}
	return false;
}

}  // namespace atom
<commit_msg>Applying changes requested by @deepak1556 after the review:<commit_after>// Copyright (c) 2015 GitHub, Inc.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

#include "atom/browser/atom_resource_dispatcher_host_delegate.h"

#include "atom/browser/atom_browser_context.h"
#include "atom/browser/login_handler.h"
#include "atom/browser/web_contents_permission_helper.h"
#include "atom/browser/web_contents_preferences.h"
#include "atom/common/atom_constants.h"
#include "atom/common/platform_util.h"
#include "base/strings/stringprintf.h"
#include "base/strings/utf_string_conversions.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/download_manager.h"
#include "content/public/browser/stream_info.h"
#include "content/public/browser/render_frame_host.h"
#include "net/base/escape.h"
#include "net/ssl/client_cert_store.h"
#include "net/url_request/url_request.h"
#include "url/gurl.h"

#if defined(USE_NSS_CERTS)
#include "net/ssl/client_cert_store_nss.h"
#elif defined(OS_WIN)
#include "net/ssl/client_cert_store_win.h"
#elif defined(OS_MACOSX)
#include "net/ssl/client_cert_store_mac.h"
#endif

using content::BrowserThread;

namespace atom {

namespace {

void OnOpenExternal(const GURL& escaped_url,
                    bool allowed) {
  if (allowed)
    platform_util::OpenExternal(
#if defined(OS_WIN)
        base::UTF8ToUTF16(escaped_url.spec()),
#else
        escaped_url,
#endif
        true);
}

void HandleExternalProtocolInUI(
    const GURL& url,
    const content::ResourceRequestInfo::WebContentsGetter& web_contents_getter,
    bool has_user_gesture) {
  content::WebContents* web_contents = web_contents_getter.Run();
  if (!web_contents)
    return;

  auto permission_helper =
      WebContentsPermissionHelper::FromWebContents(web_contents);
  if (!permission_helper)
    return;

  GURL escaped_url(net::EscapeExternalHandlerValue(url.spec()));
  auto callback = base::Bind(&OnOpenExternal, escaped_url);
  permission_helper->RequestOpenExternalPermission(callback, has_user_gesture);
}

void OnPdfResourceIntercepted(
	const GURL& original_url,
    int render_process_host_id,
    int render_frame_id,
	const content::ResourceRequestInfo::WebContentsGetter&
	web_contents_getter) {
	content::WebContents* web_contents = web_contents_getter.Run();
	if (!web_contents)
		return;

	if (!WebContentsPreferences::IsPluginsEnabled(web_contents)) {
		auto browser_context = web_contents->GetBrowserContext();
		auto download_manager =
			content::BrowserContext::GetDownloadManager(browser_context);

		download_manager->DownloadUrl(
			content::DownloadUrlParameters::CreateForWebContentsMainFrame(
				web_contents, original_url));
		return;
	}

	// The URL passes the original pdf resource url, that will be requested
	// by the webui page.
	// chrome://pdf-viewer/index.html?src=https://somepage/123.pdf
	content::NavigationController::LoadURLParams params(GURL(base::StringPrintf(
		"%sindex.html?%s=%s", kPdfViewerUIOrigin, kPdfPluginSrc,
		net::EscapeUrlEncodedData(original_url.spec(), false).c_str())));

	content::RenderFrameHost* frame_host =
		content::RenderFrameHost::FromID(render_process_host_id, render_frame_id);
	if (!frame_host)
	{
		return;
	}

	params.frame_tree_node_id = frame_host->GetFrameTreeNodeId();
	web_contents->GetController().LoadURLWithParams(params);
}

}  // namespace

AtomResourceDispatcherHostDelegate::AtomResourceDispatcherHostDelegate() {
}

bool AtomResourceDispatcherHostDelegate::HandleExternalProtocol(
    const GURL& url,
    content::ResourceRequestInfo* info) {
  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
                          base::Bind(&HandleExternalProtocolInUI,
                                     url,
                                     info->GetWebContentsGetterForRequest(),
                                     info->HasUserGesture()));
  return true;
}

content::ResourceDispatcherHostLoginDelegate*
AtomResourceDispatcherHostDelegate::CreateLoginDelegate(
    net::AuthChallengeInfo* auth_info,
    net::URLRequest* request) {
  return new LoginHandler(auth_info, request);
}

std::unique_ptr<net::ClientCertStore>
AtomResourceDispatcherHostDelegate::CreateClientCertStore(
    content::ResourceContext* resource_context) {
  #if defined(USE_NSS_CERTS)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreNSS(
        net::ClientCertStoreNSS::PasswordDelegateFactory()));
  #elif defined(OS_WIN)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreWin());
  #elif defined(OS_MACOSX)
    return std::unique_ptr<net::ClientCertStore>(new net::ClientCertStoreMac());
  #elif defined(USE_OPENSSL)
    return std::unique_ptr<net::ClientCertStore>();
  #endif
}

bool AtomResourceDispatcherHostDelegate::ShouldInterceptResourceAsStream(
	net::URLRequest* request,
	const base::FilePath& plugin_path,
	const std::string& mime_type,
	GURL* origin,
	std::string* payload) {
	const content::ResourceRequestInfo* info =
		content::ResourceRequestInfo::ForRequest(request);

	int render_process_host_id;
	int render_frame_id;
	if (!info->GetAssociatedRenderFrame(&render_process_host_id, &render_frame_id))
	{
		return false;
	}

	if (mime_type == "application/pdf") {
		*origin = GURL(kPdfViewerUIOrigin);
		content::BrowserThread::PostTask(
			BrowserThread::UI, FROM_HERE,
			base::Bind(&OnPdfResourceIntercepted, request->url(),
				render_process_host_id, render_frame_id, info->GetWebContentsGetterForRequest()));
		return true;
	}
	return false;
}

}  // namespace atom
<|endoftext|>


<commit_before><commit_msg>Updates in the direction of mixed-gaussian computations<commit_after><|endoftext|>


<commit_before>/*
 * Appcelerator Kroll - licensed under the Apache Public License 2
 * see LICENSE in the root folder for details on the license.
 * Copyright (c) 2009 Appcelerator, Inc. All Rights Reserved.
 */

#include "blob.h"
#include <cstring>
#include <Poco/String.h>
#include <Poco/StringTokenizer.h>

namespace kroll
{
	Blob::Blob() : StaticBoundObject("Blob")
	{
		CreateWithCopy(NULL, 0);
	}

	Blob::Blob(char *buf, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		// Assume this guy is null terminated. If not -- oops.
		if (makeCopy)
		{
			CreateWithCopy(buf, strlen(buf));
		}
		else
		{
			CreateWithReference(buf, strlen(buf));
		}
	}

	Blob::Blob(char *bufferIn, int len, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		if (makeCopy)
		{
			CreateWithCopy(bufferIn, len);
		}
		else
		{
			CreateWithReference(bufferIn, len);
		}
	}

	Blob::Blob(const char *bufferIn, int len, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		CreateWithCopy(bufferIn, len);
	}

	Blob::Blob(std::string str) : StaticBoundObject("Blob")
	{
		CreateWithCopy(str.c_str(), str.length());
	}

	Blob::Blob(std::string& str) : StaticBoundObject("Blob")
	{
		CreateWithCopy(str.c_str(), str.length());
	}

	void Blob::CreateWithCopy(const char *bufferIn, int length)
	{
		if (length > 0)
		{
			// Store the buffer with a null terminator so
			// that we can use it like a string later on.
			char* bufferCopy = new char[length + 1];
			memcpy(bufferCopy, bufferIn, length);
			bufferCopy[length] = '\0';
			this->CreateWithReference(bufferCopy, length);
		}
		else
		{
			this->CreateWithReference(NULL, length);
		}
	}

	void Blob::CreateWithReference(char* buffer, int length)
	{
		this->buffer = buffer;
		this->length = length;

		/**
		 * @tiapi(method=True,name=Blob.toString,since=0.3)
		 * @tiapi Return a string representation of a blob
		 * @tiresult[String] This blob as a String
		 */
		this->SetMethod("toString", &Blob::ToString);

		/**
		 * @tiapi(method=True,name=Blob.get,since=0.3)
		 * @tiapi Return a VoidPtr representation of a Blob
		 * @tiresult[VoidPtr] This blob as a VoidPtr
		 */
		this->SetMethod("get", &Blob::Get);

		// mimic some string operations to make it more
		// friendly when using a Blob in Javascript
		/**
		 * @tiapi(method=True,name=Blob.indexOf,since=0.3)
		 * @tiapi Return the index of a String within this Blob
		 * @tiarg[String, needle] The String to search for
		 * @tiresult[Number] The integer index of the String or -1 if not found
		 */
		this->SetMethod("indexOf", &Blob::IndexOf);

		/**
		 * @tiapi(method=True,name=Blob.lastIndexOf,since=0.3)
		 * @tiapi Return the last index of a String within this Blob
		 * @tiarg[String, needle] The String to search for
		 * @tiresult[Number] The last integer index of the String or -1 if not found
		 */
		this->SetMethod("lastIndexOf", &Blob::LastIndexOf);

		/**
		 * @tiapi(method=True,name=Blob.charAt,since=0.3)
		 * @tiapi Return a character representing a byte at at given index in a Blob
		 * @tiarg[Number, index] The index to look for a character at
		 * @tiresult[String] A String containing a character representing the byte at the given index
		 */
		this->SetMethod("charAt", &Blob::CharAt);

		/**
		 * @tiapi(method=True,name=Blob.split,since=0.3)
		 * @tiapi Split a blob as if it were a string given a delimiter.
		 * @tiapi This method returns empty matches. For instance:
		 * @tiapi
"abc,def,,".split(",") --> ['abc', 'def', '', '']

		 * @tiarg[String, delimiter] The index to look for a character at
		 * @tiarg[Number, limit, optional=True] The maximum number of matches to return
		 * @tiresult[Array<String>] A array containing the segments
		 */
		this->SetMethod("split", &Blob::Split);

		/**
		 * @tiapi(method=True,name=Blob.substring,since=0.3)
		 * @tiapi Return a substring of a Blob given a start index and end index
		 * @tiapi If no end index is given, return all characters from the start index
		 * @tiapi to the end of the string. If startIndex > endIndex, the indexes are swapped.
		 * @tiarg[Number, startIndex] The starting index
		 * @tiarg[Number, endIndex, optional=True] The ending index
		 * @tiresult[String] The substring between startIndex and endIndex
		 */
		this->SetMethod("substring", &Blob::Substring);

		/**
		 * @tiapi(method=True,name=Blob.substr,since=0.3)
		 * @tiapi Return a substring of a Blob given a start index and a length
		 * @tiapi If no length is given, all characters from the start to the
		 * @tiapi end of the string are returned.
		 * @tiarg[Number, startIndex] The starting index
		 * @tiarg[Number, length, optional=True] The length of the substring
		 * @tiresult[String] The substring between startIndex and the given length
		 */
		this->SetMethod("substr", &Blob::Substr);

		/**
		 * @tiapi(method=True,name=Blob.toLowerCase,since=0.3)
		 * @tiapi Convert characters in the Blob to lower case as if it were a string.
		 * @tiresult[String] The resulting String
		 */
		this->SetMethod("toLowerCase", &Blob::ToLowerCase);

		/**
		 * @tiapi(method=True,name=Blob.toUpperCase,since=0.3)
		 * @tiapi Convert characters in the Blob to upper case as if it were a string.
		 * @tiresult[String] The resulting String
		 */
		this->SetMethod("toUpperCase", &Blob::ToUpperCase);

		/**
		 * @tiapi(property=True,name=Blob.length,since=0.3) The number of bytes in this blob
		 */
		this->Set("length", Value::NewInt(length));
	}

	Blob::~Blob()
	{
		if (this->buffer!=NULL)
		{
			delete [] this->buffer;
		}
		this->buffer = NULL;
		this->length = 0;
	}

	void Blob::ToString(const ValueList& args, SharedValue result)
	{
		if (this->length == 0)
		{
			result->SetString("");
		}
		else
		{
			result->SetString(buffer);
		}
	}

	void Blob::Get(const ValueList& args, SharedValue result)
	{
		result->SetVoidPtr(buffer);
	}

	void Blob::Length(const ValueList& args, SharedValue result)
	{
		result->SetInt(length);
	}

	void Blob::IndexOf(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/indexOf
		args.VerifyException("Blob.indexOf", "s,?i");

		if (this->length <= 0)
		{
			result->SetInt(-1);
		}
		else
		{
			std::string target = this->buffer;
			std::string needle = args.at(0)->ToString();
			int start = 0;
			if (args.size() > 1)
			{
				start = args.GetNumber(1);
				if (start < 0)
				{
					start = 0;
				}
			}
			result->SetInt(target.find(needle, start));
		}
	}

	void Blob::LastIndexOf(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/lastIndexOf
		args.VerifyException("Blob.lastIndexOf", "s,?i");

		if (this->length <= 0)
		{
			result->SetInt(-1);
		}
		else
		{
			std::string target = this->buffer;
			std::string needle = args.at(0)->ToString();
			int start = target.size() + 1;
			if (args.size() > 1)
			{
				start = args.GetNumber(1);
				if (start < 0)
				{
					start = 0;
				}
			}
			result->SetInt(target.rfind(needle, start));
		}
	}

	void Blob::CharAt(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/core_javascript_1.5_reference/global_objects/string/charat
		args.VerifyException("Blob.charAt", "n");
		int position = args.at(0)->ToInt();

		char buf[2] = {'\0', '\0'};
		if (position >= 0 && position < this->length)
		{
			buf[0] = this->buffer[position];
		}
		result->SetString(buf);
	}

	void Blob::Split(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/split
		// Except support for regular expressions
		args.VerifyException("Blob.split", "?s,i");

		SharedKList list = new StaticBoundList();
		result->SetList(list);

		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}
		else
		{
			list->Append(Value::NewString(target));
			return;
		}

		if (args.size() <= 0)
		{
			list->Append(Value::NewString(target));
			return;
		}

		std::string separator = args.GetString(0);
		int limit = INT_MAX;
		if (args.size() > 1)
		{
			limit = args.GetInt(1);
		}

		// We could use Poco's tokenizer here, but it doesn't split strings
		// like "abc,def,," -> ['abc', 'def', '', ''] correctly. It produces
		// ['abc', 'def', ''] which is a different behavior than the JS split.
		// So we roll our own for now -- it's not very efficient right now, but
		// it should be correct.
		size_t next = target.find(separator);
		while (target.size() > 0 && next != std::string::npos)
		{
			std::string token;
			if (separator.size() == 0)
			{
				token = target.substr(0, 1);
			}
			else
			{
				token = target.substr(0, next);
			}
			target = target.substr(next + 1);
			next = target.find(separator);

			if ((int) list->Size() >= limit)
				return;

			list->Append(Value::NewString(token));
		}

		if ((int) list->Size() < limit && separator.size() != 0)
			list->Append(Value::NewString(target));
	}

	void Blob::Substr(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/substr
		args.VerifyException("Blob.substr", "i,?i");
		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}

		int start = args.GetInt(0);
		if (start > 0 && start >= this->length)
		{
			result->SetString("");
			return;
		}

		if (start < 0 && (-1*start) > this->length)
		{
			start = 0;
		}
		else if (start < 0)
		{
			start = this->length + start;
		}

		int length = this->length - start;
		if (args.size() > 1)
		{
			length = args.GetInt(1);
		}

		if (length <= 0)
		{
			result->SetString("");
			return;
		}

		std::string r = target.substr(start, length);
		result->SetString(r);
	}

	void Blob::Substring(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/substring
		args.VerifyException("Blob.substring", "i,?i");
		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}

		int indexA = args.GetInt(0);
		if (indexA < 0)
			indexA = 0;
		if (indexA > (int) target.size())
			indexA = target.size();

		if (args.size() < 2)
		{
			std::string r = target.substr(indexA);
			result->SetString(r);
		}
		else
		{
			int indexB = args.GetInt(1);
			if (indexB < 0)
				indexB = 0;
			if (indexB > (int) target.size())
				indexB = target.size();

			if (indexA == indexB)
			{
				result->SetString("");
				return;
			}
			if (indexA > indexB)
			{
				int temp = indexA;
				indexA = indexB;
				indexB = temp;
			}
			std::string r = target.substr(indexA, indexB - indexA);
			result->SetString(r);
		}
	}

	void Blob::ToLowerCase(const ValueList& args, SharedValue result)
	{
		if (this->length > 0)
		{
			std::string target = this->buffer;
			std::string r = Poco::toLower(target);
			result->SetString(r);
		}
		else
		{
			result->SetNull();
		}
	}

	void Blob::ToUpperCase(const ValueList& args, SharedValue result)
	{
		if (this->length > 0)
		{
			std::string target = this->buffer;
			std::string r = Poco::toUpper(target);
			result->SetString(r);
		}
		else
		{
			result->SetNull();
		}
	}

	AutoBlob Blob::GlobBlobs(std::vector<AutoBlob>& blobs)
	{
		int size = 0;
		for (size_t i = 0; i < blobs.size(); i++)
		{
			size += blobs.at(i)->Length();
		}

		char* buffer = new char[size+1];
		buffer[size] = '\0';

		char* current = buffer;
		for (size_t i = 0; i < blobs.size(); i++)
		{
			AutoBlob blob = blobs.at(i);
			if (blob->Length() > 0)
			{
				memcpy(current, blob->Get(), blob->Length());
				current += blob->Length() + 1;
			}
		}

		return new Blob(buffer, size, false);
	}
}
<commit_msg>just use blob length in concat, avoid null separators<commit_after>/*
 * Appcelerator Kroll - licensed under the Apache Public License 2
 * see LICENSE in the root folder for details on the license.
 * Copyright (c) 2009 Appcelerator, Inc. All Rights Reserved.
 */

#include "blob.h"
#include <cstring>
#include <Poco/String.h>
#include <Poco/StringTokenizer.h>

namespace kroll
{
	Blob::Blob() : StaticBoundObject("Blob")
	{
		CreateWithCopy(NULL, 0);
	}

	Blob::Blob(char *buf, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		// Assume this guy is null terminated. If not -- oops.
		if (makeCopy)
		{
			CreateWithCopy(buf, strlen(buf));
		}
		else
		{
			CreateWithReference(buf, strlen(buf));
		}
	}

	Blob::Blob(char *bufferIn, int len, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		if (makeCopy)
		{
			CreateWithCopy(bufferIn, len);
		}
		else
		{
			CreateWithReference(bufferIn, len);
		}
	}

	Blob::Blob(const char *bufferIn, int len, bool makeCopy) :
		StaticBoundObject("Blob")
	{
		CreateWithCopy(bufferIn, len);
	}

	Blob::Blob(std::string str) : StaticBoundObject("Blob")
	{
		CreateWithCopy(str.c_str(), str.length());
	}

	Blob::Blob(std::string& str) : StaticBoundObject("Blob")
	{
		CreateWithCopy(str.c_str(), str.length());
	}

	void Blob::CreateWithCopy(const char *bufferIn, int length)
	{
		if (length > 0)
		{
			// Store the buffer with a null terminator so
			// that we can use it like a string later on.
			char* bufferCopy = new char[length + 1];
			memcpy(bufferCopy, bufferIn, length);
			bufferCopy[length] = '\0';
			this->CreateWithReference(bufferCopy, length);
		}
		else
		{
			this->CreateWithReference(NULL, length);
		}
	}

	void Blob::CreateWithReference(char* buffer, int length)
	{
		this->buffer = buffer;
		this->length = length;

		/**
		 * @tiapi(method=True,name=Blob.toString,since=0.3)
		 * @tiapi Return a string representation of a blob
		 * @tiresult[String] This blob as a String
		 */
		this->SetMethod("toString", &Blob::ToString);

		/**
		 * @tiapi(method=True,name=Blob.get,since=0.3)
		 * @tiapi Return a VoidPtr representation of a Blob
		 * @tiresult[VoidPtr] This blob as a VoidPtr
		 */
		this->SetMethod("get", &Blob::Get);

		// mimic some string operations to make it more
		// friendly when using a Blob in Javascript
		/**
		 * @tiapi(method=True,name=Blob.indexOf,since=0.3)
		 * @tiapi Return the index of a String within this Blob
		 * @tiarg[String, needle] The String to search for
		 * @tiresult[Number] The integer index of the String or -1 if not found
		 */
		this->SetMethod("indexOf", &Blob::IndexOf);

		/**
		 * @tiapi(method=True,name=Blob.lastIndexOf,since=0.3)
		 * @tiapi Return the last index of a String within this Blob
		 * @tiarg[String, needle] The String to search for
		 * @tiresult[Number] The last integer index of the String or -1 if not found
		 */
		this->SetMethod("lastIndexOf", &Blob::LastIndexOf);

		/**
		 * @tiapi(method=True,name=Blob.charAt,since=0.3)
		 * @tiapi Return a character representing a byte at at given index in a Blob
		 * @tiarg[Number, index] The index to look for a character at
		 * @tiresult[String] A String containing a character representing the byte at the given index
		 */
		this->SetMethod("charAt", &Blob::CharAt);

		/**
		 * @tiapi(method=True,name=Blob.split,since=0.3)
		 * @tiapi Split a blob as if it were a string given a delimiter.
		 * @tiapi This method returns empty matches. For instance:
		 * @tiapi
"abc,def,,".split(",") --> ['abc', 'def', '', '']

		 * @tiarg[String, delimiter] The index to look for a character at
		 * @tiarg[Number, limit, optional=True] The maximum number of matches to return
		 * @tiresult[Array<String>] A array containing the segments
		 */
		this->SetMethod("split", &Blob::Split);

		/**
		 * @tiapi(method=True,name=Blob.substring,since=0.3)
		 * @tiapi Return a substring of a Blob given a start index and end index
		 * @tiapi If no end index is given, return all characters from the start index
		 * @tiapi to the end of the string. If startIndex > endIndex, the indexes are swapped.
		 * @tiarg[Number, startIndex] The starting index
		 * @tiarg[Number, endIndex, optional=True] The ending index
		 * @tiresult[String] The substring between startIndex and endIndex
		 */
		this->SetMethod("substring", &Blob::Substring);

		/**
		 * @tiapi(method=True,name=Blob.substr,since=0.3)
		 * @tiapi Return a substring of a Blob given a start index and a length
		 * @tiapi If no length is given, all characters from the start to the
		 * @tiapi end of the string are returned.
		 * @tiarg[Number, startIndex] The starting index
		 * @tiarg[Number, length, optional=True] The length of the substring
		 * @tiresult[String] The substring between startIndex and the given length
		 */
		this->SetMethod("substr", &Blob::Substr);

		/**
		 * @tiapi(method=True,name=Blob.toLowerCase,since=0.3)
		 * @tiapi Convert characters in the Blob to lower case as if it were a string.
		 * @tiresult[String] The resulting String
		 */
		this->SetMethod("toLowerCase", &Blob::ToLowerCase);

		/**
		 * @tiapi(method=True,name=Blob.toUpperCase,since=0.3)
		 * @tiapi Convert characters in the Blob to upper case as if it were a string.
		 * @tiresult[String] The resulting String
		 */
		this->SetMethod("toUpperCase", &Blob::ToUpperCase);

		/**
		 * @tiapi(property=True,name=Blob.length,since=0.3) The number of bytes in this blob
		 */
		this->Set("length", Value::NewInt(length));
	}

	Blob::~Blob()
	{
		if (this->buffer!=NULL)
		{
			delete [] this->buffer;
		}
		this->buffer = NULL;
		this->length = 0;
	}

	void Blob::ToString(const ValueList& args, SharedValue result)
	{
		if (this->length == 0)
		{
			result->SetString("");
		}
		else
		{
			result->SetString(buffer);
		}
	}

	void Blob::Get(const ValueList& args, SharedValue result)
	{
		result->SetVoidPtr(buffer);
	}

	void Blob::Length(const ValueList& args, SharedValue result)
	{
		result->SetInt(length);
	}

	void Blob::IndexOf(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/indexOf
		args.VerifyException("Blob.indexOf", "s,?i");

		if (this->length <= 0)
		{
			result->SetInt(-1);
		}
		else
		{
			std::string target = this->buffer;
			std::string needle = args.at(0)->ToString();
			int start = 0;
			if (args.size() > 1)
			{
				start = args.GetNumber(1);
				if (start < 0)
				{
					start = 0;
				}
			}
			result->SetInt(target.find(needle, start));
		}
	}

	void Blob::LastIndexOf(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/lastIndexOf
		args.VerifyException("Blob.lastIndexOf", "s,?i");

		if (this->length <= 0)
		{
			result->SetInt(-1);
		}
		else
		{
			std::string target = this->buffer;
			std::string needle = args.at(0)->ToString();
			int start = target.size() + 1;
			if (args.size() > 1)
			{
				start = args.GetNumber(1);
				if (start < 0)
				{
					start = 0;
				}
			}
			result->SetInt(target.rfind(needle, start));
		}
	}

	void Blob::CharAt(const ValueList& args, SharedValue result)
	{
		// https://developer.mozilla.org/en/core_javascript_1.5_reference/global_objects/string/charat
		args.VerifyException("Blob.charAt", "n");
		int position = args.at(0)->ToInt();

		char buf[2] = {'\0', '\0'};
		if (position >= 0 && position < this->length)
		{
			buf[0] = this->buffer[position];
		}
		result->SetString(buf);
	}

	void Blob::Split(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/split
		// Except support for regular expressions
		args.VerifyException("Blob.split", "?s,i");

		SharedKList list = new StaticBoundList();
		result->SetList(list);

		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}
		else
		{
			list->Append(Value::NewString(target));
			return;
		}

		if (args.size() <= 0)
		{
			list->Append(Value::NewString(target));
			return;
		}

		std::string separator = args.GetString(0);
		int limit = INT_MAX;
		if (args.size() > 1)
		{
			limit = args.GetInt(1);
		}

		// We could use Poco's tokenizer here, but it doesn't split strings
		// like "abc,def,," -> ['abc', 'def', '', ''] correctly. It produces
		// ['abc', 'def', ''] which is a different behavior than the JS split.
		// So we roll our own for now -- it's not very efficient right now, but
		// it should be correct.
		size_t next = target.find(separator);
		while (target.size() > 0 && next != std::string::npos)
		{
			std::string token;
			if (separator.size() == 0)
			{
				token = target.substr(0, 1);
			}
			else
			{
				token = target.substr(0, next);
			}
			target = target.substr(next + 1);
			next = target.find(separator);

			if ((int) list->Size() >= limit)
				return;

			list->Append(Value::NewString(token));
		}

		if ((int) list->Size() < limit && separator.size() != 0)
			list->Append(Value::NewString(target));
	}

	void Blob::Substr(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/substr
		args.VerifyException("Blob.substr", "i,?i");
		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}

		int start = args.GetInt(0);
		if (start > 0 && start >= this->length)
		{
			result->SetString("");
			return;
		}

		if (start < 0 && (-1*start) > this->length)
		{
			start = 0;
		}
		else if (start < 0)
		{
			start = this->length + start;
		}

		int length = this->length - start;
		if (args.size() > 1)
		{
			length = args.GetInt(1);
		}

		if (length <= 0)
		{
			result->SetString("");
			return;
		}

		std::string r = target.substr(start, length);
		result->SetString(r);
	}

	void Blob::Substring(const ValueList& args, SharedValue result)
	{
		// This method now follows the spec located at:
		// https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Objects/String/substring
		args.VerifyException("Blob.substring", "i,?i");
		std::string target = "";
		if (this->length > 0)
		{
			target = this->buffer;
		}

		int indexA = args.GetInt(0);
		if (indexA < 0)
			indexA = 0;
		if (indexA > (int) target.size())
			indexA = target.size();

		if (args.size() < 2)
		{
			std::string r = target.substr(indexA);
			result->SetString(r);
		}
		else
		{
			int indexB = args.GetInt(1);
			if (indexB < 0)
				indexB = 0;
			if (indexB > (int) target.size())
				indexB = target.size();

			if (indexA == indexB)
			{
				result->SetString("");
				return;
			}
			if (indexA > indexB)
			{
				int temp = indexA;
				indexA = indexB;
				indexB = temp;
			}
			std::string r = target.substr(indexA, indexB - indexA);
			result->SetString(r);
		}
	}

	void Blob::ToLowerCase(const ValueList& args, SharedValue result)
	{
		if (this->length > 0)
		{
			std::string target = this->buffer;
			std::string r = Poco::toLower(target);
			result->SetString(r);
		}
		else
		{
			result->SetNull();
		}
	}

	void Blob::ToUpperCase(const ValueList& args, SharedValue result)
	{
		if (this->length > 0)
		{
			std::string target = this->buffer;
			std::string r = Poco::toUpper(target);
			result->SetString(r);
		}
		else
		{
			result->SetNull();
		}
	}

	AutoBlob Blob::GlobBlobs(std::vector<AutoBlob>& blobs)
	{
		int size = 0;
		for (size_t i = 0; i < blobs.size(); i++)
		{
			size += blobs.at(i)->Length();
		}

		char* buffer = new char[size+1];
		buffer[size] = '\0';

		char* current = buffer;
		for (size_t i = 0; i < blobs.size(); i++)
		{
			AutoBlob blob = blobs.at(i);
			if (blob->Length() > 0)
			{
				memcpy(current, blob->Get(), blob->Length());
				current += blob->Length();
			}
		}

		return new Blob(buffer, size, false);
	}
}
<|endoftext|>


<commit_before><commit_msg>Comments?<commit_after><|endoftext|>


<commit_before>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once
#ifndef NET_IP6_ADDR_HPP
#define NET_IP6_ADDR_HPP

#include <common>
#include <net/util.hpp>
#include <cstdlib>
#include <string>
#include <hw/mac_addr.hpp>

namespace net {
namespace ip6 {

/**
 * This type is thrown when creating an instance of Addr
 * with a std::string that doesn't represent a valid IPv6
 * Address
 */
struct Invalid_Address : public std::runtime_error {
  using runtime_error::runtime_error;
}; //< struct Invalid_Address

/**
 * IPv6 Address representation
 */
struct Addr {
  constexpr Addr() noexcept
    : i32{{0, 0, 0, 0}} {}

  Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2,
       uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2)
  {
    i16[0] = htons(a1); i16[1] = htons(a2);
    i16[2] = htons(b1); i16[3] = htons(b2);
    i16[4] = htons(c1); i16[5] = htons(c2);
    i16[6] = htons(d1); i16[7] = htons(d2);
  }

  explicit Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
  {
    i32[0] = htonl(a); i32[1] = htonl(b);
    i32[2] = htonl(c); i32[3] = htonl(d);
  }

  Addr(const Addr& a) noexcept
    : i64{a.i64} {}

  Addr(Addr&& a) noexcept
    : i64{a.i64} {}

  // returns this IPv6 Address as a string
  std::string str() const {
    char ipv6_addr[40];
    snprintf(ipv6_addr, sizeof(ipv6_addr),
            "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x",
            ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]),
            ntohs(i16[3]), ntohs(i16[4]), ntohs(i16[5]),
            ntohs(i16[6]), ntohs(i16[7]));
    return ipv6_addr;
  }

  /**
   * Get a string representation of this type
   *
   * @return A string representation of this type
   */
  std::string to_string() const
  { return str(); }

  // multicast IPv6 Addresses
  static const Addr node_all_nodes;     // RFC 4921
  static const Addr node_all_routers;   // RFC 4921
  static const Addr node_mDNSv6;        // RFC 6762 (multicast DNSv6)

  // unspecified link-local Address
  static const Addr link_unspecified;
  static const Addr addr_any;

  // RFC 4291  2.4.6:
  // Link-Local Addresses are designed to be used for Addressing on a
  // neighbor discovery, or when no routers are present.
  // single link for purposes such as automatic Address configuration,
  static const Addr link_all_nodes;     // RFC 4921
  static const Addr link_all_routers;   // RFC 4921
  static const Addr link_mDNSv6;        // RFC 6762

  static const Addr link_dhcp_servers;  // RFC 3315
  static const Addr site_dhcp_servers;  // RFC 3315

  // returns true if this Addr is a IPv6 multicast Address
  bool is_multicast() const
  {
    /**
       RFC 4291 2.7 Multicast Addresses

       An IPv6 multicast Address is an identifier for a group of interfaces
       (typically on different nodes). An interface may belong to any
       number of multicast groups. Multicast Addresses have the following format:
       |   8    |  4 |  4 |                  112 bits                   |
       +------ -+----+----+---------------------------------------------+
       |11111111|flgs|scop|                  group ID                   |
       +--------+----+----+---------------------------------------------+
    **/
    return ((ntohs(i16[0]) & 0xFF00) == 0xFF00);
  }

  bool is_linklocal() const
  {
    return ((ntohs(i16[0]) & 0xFE80) == 0xFE80);
  }

  bool is_solicit_multicast() const
  {
      return ((ntohs(i32[0]) ^  (0xff020000)) |
              ntohs(i32[1]) |
              (ntohs(i32[2]) ^ (0x00000001)) |
              (ntohs(i32[3]) ^ 0xff)) == 0;
  }

  uint8_t* data()
  {
      return reinterpret_cast<uint8_t*> (i16.data());
  }

  Addr& solicit(const Addr other) noexcept {
    i32[0] = htonl(0xFF020000);
    i32[1] = 0;
    i32[2] = htonl(0x1);
    i32[3] = htonl(0xFF000000) | other.i32[3];
    return *this;
  }

  /**
   *
   **/
  bool is_loopback() const noexcept
  { return i32[0] == 0 && i32[1] == 0 &&
      i32[2] == 0 && ntohl(i32[3]) == 1; }

  template <typename T>
  T get_part(const uint8_t n) const
  {
     static_assert(std::is_same_v<T, uint8_t> or
             std::is_same_v<T, uint16_t> or
             std::is_same_v<T, uint32_t> or
             std::is_same_v<T, uint64_t>, "Unallowed T");

     if constexpr (std::is_same_v<T, uint8_t>) {
         Expects(n < 16);
         return i8[n];
     } else if constexpr (std::is_same_v<T, uint16_t>) {
         Expects(n < 8);
         return i16[n];
     } else if constexpr (std::is_same_v<T, uint32_t>) {
         Expects(n < 4);
         return i32[n];
     } else {
         Expects(n < 2);
         return i64[n];
     }
  }

  template <typename T>
  void set_part(const uint8_t n, T val)
  {
     static_assert(std::is_same_v<T, uint8_t> or
             std::is_same_v<T, uint16_t> or
             std::is_same_v<T, uint32_t> or
             std::is_same_v<T, uint64_t>, "Unallowed T");

     if constexpr (std::is_same_v<T, uint8_t>) {
         Expects(n < 16);
         i8[n] = val;
     } else if constexpr (std::is_same_v<T, uint16_t>) {
         Expects(n < 8);
         i16[n] = val;
     } else if constexpr (std::is_same_v<T, uint32_t>) {
         Expects(n < 4);
         i32[n] = val;
     } else {
         Expects(n < 2);
         i64[n] = val;
     }
  }

  void set(const MAC::Addr &mac, const uint8_t loc = 0)
  {
      Expects(loc <= (16 - 6));
      uint8_t start_loc = (10 - loc);

      for (int i = 0; i < 6; i++) {
        i8[start_loc++] = mac[i];
      }
  }

  /**
   * Assignment operator
   */
  Addr& operator=(const Addr& other) noexcept
  {
    i64 = other.i64;
    return *this;
  }

  Addr& operator=(Addr&& other) noexcept
  {
    i64 = other.i64;
    return *this;
  }

  /**
   * Operator to check for equality
   */
  bool operator==(const Addr& other) const noexcept
  { return i64 == other.i64; }

  /**
   * Operator to check for inequality
   */
  bool operator!=(const Addr& other) const noexcept
  { return not (*this == other); }

  /**
   * Operator to check for greater-than relationship
   */
  bool operator>(const Addr& other) const noexcept
  {
    if(ntohl(i32[0]) > ntohl(other.i32[0])) return true;
    if(ntohl(i32[1]) > ntohl(other.i32[1])) return true;
    if(ntohl(i32[2]) > ntohl(other.i32[2])) return true;
    if(ntohl(i32[3]) > ntohl(other.i32[3])) return true;
    return false;
  }

  /**
   * Operator to check for greater-than-or-equal relationship
   */
  bool operator>=(const Addr& other) const noexcept
  { return (*this > other or *this == other); }

  /**
   * Operator to check for lesser-than relationship
   */
  bool operator<(const Addr& other) const noexcept
  { return not (*this >= other); }

  /**
   * Operator to check for lesser-than-or-equal relationship
   */
  bool operator<=(const Addr& other) const noexcept
  { return (*this < other or *this == other); }

  /**
   * Operator to perform a bitwise-and operation on the given
   * IPv6 addresses
   */
  Addr operator&(const Addr& other) const noexcept
  { return Addr{i32[0] & other.i32[0],
               i32[1] & other.i32[1],
               i32[2] & other.i32[2],
               i32[3] & other.i32[3] }; }

  Addr operator&(uint8_t prefix) const noexcept
  {
      int i = 3;
      uint8_t mask;
      uint32_t addr[32];

      addr[0] = htonl(i32[0]);
      addr[1] = htonl(i32[1]);
      addr[2] = htonl(i32[2]);
      addr[3] = htonl(i32[3]);

      if (prefix > 128) {
          prefix = 128;
      }

      mask = 128 - prefix;
      while (mask >= 32) {
          addr[i--] = 0;
          mask -= 32;
      }

      if (mask != 0) {
          addr[i] &= (0xFFFFFFFF << mask);
      }
      return Addr { addr[0], addr[1],
               addr[2], addr[3] };
  }

  Addr operator|(const Addr& other) const noexcept
  { return Addr{i32[0] | other.i32[0],
               i32[1] | other.i32[1],
               i32[2] | other.i32[2],
               i32[3] | other.i32[3] }; }

  Addr operator~() const noexcept
  { return Addr{~i32[0], ~i32[1], ~i32[2], ~i32[3]}; }

  union
  {
    std::array<uint64_t, 2> i64;
    std::array<uint32_t ,4> i32;
    std::array<uint16_t, 8> i16;
    std::array<uint8_t, 16> i8;
  };
} __attribute__((packed)); //< struct Addr
static_assert(sizeof(Addr) == 16);

} //< namespace ip6
} //< namespace net

// Allow an IPv6 address to be used as key in e.g. std::unordered_map
namespace std {
  template<>
  struct hash<net::ip6::Addr> {
    size_t operator()(const net::ip6::Addr& addr) const {
      // This is temporary. Use a proper hash
      return std::hash<uint64_t>{}(addr.i64[0] + addr.i64[1]);
    }
  };
} //< namespace std
#endif
<commit_msg>ip6: Create link local address from eui64<commit_after>// This file is a part of the IncludeOS unikernel - www.includeos.org
//
// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
// and Alfred Bratterud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once
#ifndef NET_IP6_ADDR_HPP
#define NET_IP6_ADDR_HPP

#include <common>
#include <net/util.hpp>
#include <cstdlib>
#include <string>
#include <hw/mac_addr.hpp>

namespace net {
namespace ip6 {

/**
 * This type is thrown when creating an instance of Addr
 * with a std::string that doesn't represent a valid IPv6
 * Address
 */
struct Invalid_Address : public std::runtime_error {
  using runtime_error::runtime_error;
}; //< struct Invalid_Address

/**
 * IPv6 Address representation
 */
struct Addr {
  constexpr Addr() noexcept
    : i32{{0, 0, 0, 0}} {}

  Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2,
       uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2)
    : i16{htons(a1), htons(a2), htons(b1), htons(b2),
          htons(c1), htons(c2), htons(d1), htons(d2)}
  {}

  explicit Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
    : i32{htonl(a), htonl(b), htonl(c), htonl(d)}
  {}

  explicit Addr(uint64_t a, uint64_t b)
    : i64{htonll(a), htonll(b)}
  {}

  Addr(const Addr& a) noexcept
    : i64{a.i64} {}

  Addr(Addr&& a) noexcept
    : i64{a.i64} {}

  // returns this IPv6 Address as a string
  std::string str() const {
    char ipv6_addr[40];
    snprintf(ipv6_addr, sizeof(ipv6_addr),
            "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x",
            ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]),
            ntohs(i16[3]), ntohs(i16[4]), ntohs(i16[5]),
            ntohs(i16[6]), ntohs(i16[7]));
    return ipv6_addr;
  }

  /**
   * Get a string representation of this type
   *
   * @return A string representation of this type
   */
  std::string to_string() const
  { return str(); }

  // multicast IPv6 Addresses
  static const Addr node_all_nodes;     // RFC 4921
  static const Addr node_all_routers;   // RFC 4921
  static const Addr node_mDNSv6;        // RFC 6762 (multicast DNSv6)

  // unspecified link-local Address
  static const Addr link_unspecified;
  static const Addr addr_any;

  // RFC 4291  2.4.6:
  // Link-Local Addresses are designed to be used for Addressing on a
  // neighbor discovery, or when no routers are present.
  // single link for purposes such as automatic Address configuration,
  static const Addr link_all_nodes;     // RFC 4921
  static const Addr link_all_routers;   // RFC 4921
  static const Addr link_mDNSv6;        // RFC 6762

  static const Addr link_dhcp_servers;  // RFC 3315
  static const Addr site_dhcp_servers;  // RFC 3315

  // returns true if this Addr is a IPv6 multicast Address
  bool is_multicast() const
  {
    /**
       RFC 4291 2.7 Multicast Addresses

       An IPv6 multicast Address is an identifier for a group of interfaces
       (typically on different nodes). An interface may belong to any
       number of multicast groups. Multicast Addresses have the following format:
       |   8    |  4 |  4 |                  112 bits                   |
       +------ -+----+----+---------------------------------------------+
       |11111111|flgs|scop|                  group ID                   |
       +--------+----+----+---------------------------------------------+
    **/
    return ((ntohs(i16[0]) & 0xFF00) == 0xFF00);
  }

  bool is_linklocal() const
  {
    return ((ntohs(i16[0]) & 0xFE80) == 0xFE80);
  }

  bool is_solicit_multicast() const
  {
      return ((ntohs(i32[0]) ^  (0xff020000)) |
              ntohs(i32[1]) |
              (ntohs(i32[2]) ^ (0x00000001)) |
              (ntohs(i32[3]) ^ 0xff)) == 0;
  }

  uint8_t* data()
  {
      return reinterpret_cast<uint8_t*> (i16.data());
  }

  Addr& solicit(const Addr other) noexcept {
    i32[0] = htonl(0xFF020000);
    i32[1] = 0;
    i32[2] = htonl(0x1);
    i32[3] = htonl(0xFF000000) | other.i32[3];
    return *this;
  }

  static Addr link_local(uint64_t eui) noexcept
  {
    return Addr{0xFE80'0000'0000'0000, ntohll(eui)};
  }

  /**
   *
   **/
  bool is_loopback() const noexcept
  { return i32[0] == 0 && i32[1] == 0 &&
      i32[2] == 0 && ntohl(i32[3]) == 1; }

  template <typename T>
  T get_part(const uint8_t n) const
  {
     static_assert(std::is_same_v<T, uint8_t> or
             std::is_same_v<T, uint16_t> or
             std::is_same_v<T, uint32_t> or
             std::is_same_v<T, uint64_t>, "Unallowed T");

     if constexpr (std::is_same_v<T, uint8_t>) {
         Expects(n < 16);
         return i8[n];
     } else if constexpr (std::is_same_v<T, uint16_t>) {
         Expects(n < 8);
         return i16[n];
     } else if constexpr (std::is_same_v<T, uint32_t>) {
         Expects(n < 4);
         return i32[n];
     } else {
         Expects(n < 2);
         return i64[n];
     }
  }

  template <typename T>
  void set_part(const uint8_t n, T val)
  {
     static_assert(std::is_same_v<T, uint8_t> or
             std::is_same_v<T, uint16_t> or
             std::is_same_v<T, uint32_t> or
             std::is_same_v<T, uint64_t>, "Unallowed T");

     if constexpr (std::is_same_v<T, uint8_t>) {
         Expects(n < 16);
         i8[n] = val;
     } else if constexpr (std::is_same_v<T, uint16_t>) {
         Expects(n < 8);
         i16[n] = val;
     } else if constexpr (std::is_same_v<T, uint32_t>) {
         Expects(n < 4);
         i32[n] = val;
     } else {
         Expects(n < 2);
         i64[n] = val;
     }
  }

  void set(const MAC::Addr &mac, const uint8_t loc = 0)
  {
      Expects(loc <= (16 - 6));
      uint8_t start_loc = (10 - loc);

      for (int i = 0; i < 6; i++) {
        i8[start_loc++] = mac[i];
      }
  }

  /**
   * Assignment operator
   */
  Addr& operator=(const Addr& other) noexcept
  {
    i64 = other.i64;
    return *this;
  }

  Addr& operator=(Addr&& other) noexcept
  {
    i64 = other.i64;
    return *this;
  }

  /**
   * Operator to check for equality
   */
  bool operator==(const Addr& other) const noexcept
  { return i64 == other.i64; }

  /**
   * Operator to check for inequality
   */
  bool operator!=(const Addr& other) const noexcept
  { return not (*this == other); }

  /**
   * Operator to check for greater-than relationship
   */
  bool operator>(const Addr& other) const noexcept
  {
    if(ntohl(i32[0]) > ntohl(other.i32[0])) return true;
    if(ntohl(i32[1]) > ntohl(other.i32[1])) return true;
    if(ntohl(i32[2]) > ntohl(other.i32[2])) return true;
    if(ntohl(i32[3]) > ntohl(other.i32[3])) return true;
    return false;
  }

  /**
   * Operator to check for greater-than-or-equal relationship
   */
  bool operator>=(const Addr& other) const noexcept
  { return (*this > other or *this == other); }

  /**
   * Operator to check for lesser-than relationship
   */
  bool operator<(const Addr& other) const noexcept
  { return not (*this >= other); }

  /**
   * Operator to check for lesser-than-or-equal relationship
   */
  bool operator<=(const Addr& other) const noexcept
  { return (*this < other or *this == other); }

  /**
   * Operator to perform a bitwise-and operation on the given
   * IPv6 addresses
   */
  Addr operator&(const Addr& other) const noexcept
  { return Addr{i32[0] & other.i32[0],
               i32[1] & other.i32[1],
               i32[2] & other.i32[2],
               i32[3] & other.i32[3] }; }

  Addr operator&(uint8_t prefix) const noexcept
  {
      int i = 3;
      uint8_t mask;
      uint32_t addr[32];

      addr[0] = htonl(i32[0]);
      addr[1] = htonl(i32[1]);
      addr[2] = htonl(i32[2]);
      addr[3] = htonl(i32[3]);

      if (prefix > 128) {
          prefix = 128;
      }

      mask = 128 - prefix;
      while (mask >= 32) {
          addr[i--] = 0;
          mask -= 32;
      }

      if (mask != 0) {
          addr[i] &= (0xFFFFFFFF << mask);
      }
      return Addr { addr[0], addr[1],
               addr[2], addr[3] };
  }

  Addr operator|(const Addr& other) const noexcept
  { return Addr{i32[0] | other.i32[0],
               i32[1] | other.i32[1],
               i32[2] | other.i32[2],
               i32[3] | other.i32[3] }; }

  Addr operator~() const noexcept
  { return Addr{~i32[0], ~i32[1], ~i32[2], ~i32[3]}; }

  union
  {
    std::array<uint64_t, 2> i64;
    std::array<uint32_t ,4> i32;
    std::array<uint16_t, 8> i16;
    std::array<uint8_t, 16> i8;
  };
} __attribute__((packed)); //< struct Addr
static_assert(sizeof(Addr) == 16);

} //< namespace ip6
} //< namespace net

// Allow an IPv6 address to be used as key in e.g. std::unordered_map
namespace std {
  template<>
  struct hash<net::ip6::Addr> {
    size_t operator()(const net::ip6::Addr& addr) const {
      // This is temporary. Use a proper hash
      return std::hash<uint64_t>{}(addr.i64[0] + addr.i64[1]);
    }
  };
} //< namespace std
#endif
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <memory>

#include "dll/conv_dbn.hpp"
#include "dll/ocv_visualizer.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

int main(int argc, char* argv[]){
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "load"){
            load = true;
        }
    }

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(5000);

    if(dataset.training_images.empty() || dataset.training_labels.empty()){
        return 1;
    }

    mnist::binarize_dataset(dataset);

    typedef dll::conv_dbn_desc<
        dll::dbn_layers<
            dll::conv_rbm_desc<28, 12, 40, dll::momentum, dll::batch_size<50>>::rbm_t,
            dll::conv_rbm_desc<12, 6, 40, dll::momentum, dll::batch_size<50>>::rbm_t
        >, dll::watcher<dll::opencv_dbn_visualizer>>::dbn_t dbn_t;

    auto dbn = std::make_unique<dbn_t>();

    dbn->display();

    if(load){
        std::cout << "Load from file" << std::endl;

        std::ifstream is("dbn.dat", std::ifstream::binary);
        dbn->load(is);
    } else {
        std::cout << "Start pretraining" << std::endl;
        dbn->pretrain(dataset.training_images, 5);

        std::ofstream os("dbn.dat", std::ofstream::binary);
        dbn->store(os);
    }

    return 0;
}
<commit_msg>Use weight decay<commit_after>//=======================================================================
// Copyright (c) 2014 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#include <iostream>
#include <memory>

#include "dll/conv_dbn.hpp"
#include "dll/ocv_visualizer.hpp"

#include "mnist/mnist_reader.hpp"
#include "mnist/mnist_utils.hpp"

int main(int argc, char* argv[]){
    auto load = false;

    for(int i = 1; i < argc; ++i){
        std::string command(argv[i]);

        if(command == "load"){
            load = true;
        }
    }

    auto dataset = mnist::read_dataset<std::vector, std::vector, double>(5000);

    if(dataset.training_images.empty() || dataset.training_labels.empty()){
        return 1;
    }

    mnist::binarize_dataset(dataset);

    typedef dll::conv_dbn_desc<
        dll::dbn_layers<
            dll::conv_rbm_desc<28, 12, 40, dll::momentum, dll::batch_size<50>, dll::weight_decay<dll::decay_type::L2>>::rbm_t,
            dll::conv_rbm_desc<12, 6, 40, dll::momentum, dll::batch_size<50>, dll::weight_decay<dll::decay_type::L2>>::rbm_t
        >, dll::watcher<dll::opencv_dbn_visualizer>>::dbn_t dbn_t;

    auto dbn = std::make_unique<dbn_t>();

    dbn->display();

    if(load){
        std::cout << "Load from file" << std::endl;

        std::ifstream is("dbn.dat", std::ifstream::binary);
        dbn->load(is);
    } else {
        std::cout << "Start pretraining" << std::endl;
        dbn->pretrain(dataset.training_images, 5);

        std::ofstream os("dbn.dat", std::ofstream::binary);
        dbn->store(os);
    }

    return 0;
}
<|endoftext|>


<commit_before>/// \file dtsc2mp4.cpp
/// Contains the code that will transform any valid DTSC input into valid MP4s.

#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <cstdlib>
#include <cstdio>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <mist/json.h>
#include <mist/dtsc.h> //DTSC support
#include <mist/mp4.h> //MP4 support
#include <mist/ts_packet.h> //TS support ff
#include <mist/config.h>

///\brief Holds everything unique to converters.
namespace Converters {

  ///\brief Converts DTSC from file to MP4 on stdout.
  ///\return The return code for the converter.
  int DTSC2MP4(Util::Config & conf){
    DTSC::File input(conf.getString("filename"));
    //ftyp box
    /// \todo fill ftyp with non hardcoded values from file
    MP4::FTYP ftypBox;
    ftypBox.setMajorBrand(0x69736f6d);
    ftypBox.setMinorVersion(512);
    ftypBox.setCompatibleBrands(0x69736f6d,0);
    ftypBox.setCompatibleBrands(0x69736f32,1);
    ftypBox.setCompatibleBrands(0x61766331,2);
    ftypBox.setCompatibleBrands(0x6d703431,3);
    std::cout << std::string(ftypBox.asBox(),ftypBox.boxedSize());

    //moov box
    MP4::MOOV moovBox;
      MP4::MVHD mvhdBox;
      mvhdBox.setCreationTime(0);
      mvhdBox.setModificationTime(0);
      mvhdBox.setTimeScale(1000);
      mvhdBox.setDuration(input.getMeta()["lastms"].asInt());
      /// \todo mvhd setTrackID automatic fix (next track ID)
      moovBox.setContent(mvhdBox, 0);

      //start arbitrary track addition
      int boxOffset = 1;
      for (JSON::ObjIter it = input.getMeta()["tracks"].ObjBegin(); it != input.getMeta()["tracks"].ObjEnd(); it++){
        MP4::TRAK trakBox;
          MP4::TKHD tkhdBox;
          //std::cerr << it->second["trackid"].asInt() << std::endl;
          tkhdBox.setTrackID(it->second["trackid"].asInt());

          if (it->second["type"].asString() == "video"){
            tkhdBox.setWidth(it->second["width"].asInt() << 16);
            tkhdBox.setHeight(it->second["height"].asInt() << 16);
          }
          trakBox.setContent(tkhdBox, 0);

          MP4::MDIA mdiaBox;
            MP4::MDHD mdhdBox;/// \todo fix constructor mdhd in lib
            mdhdBox.setCreationTime(0);
            mdhdBox.setModificationTime(0);
            mdhdBox.setTimeScale(1000);
            mdhdBox.setDuration(input.getMeta()["lastms"].asInt());
            mdiaBox.setContent(mdhdBox, 0);

            std::string tmpStr = it->second["type"].asString();
            MP4::HDLR hdlrBox;/// \todo fix constructor hdlr in lib
            if (tmpStr == "video"){
              hdlrBox.setHandlerType(0x76696465);//vide
            }else if (tmpStr == "audio"){
              hdlrBox.setHandlerType(0x736F756E);//soun
            }
            hdlrBox.setName(it->first);
            mdiaBox.setContent(hdlrBox, 1);

            MP4::MINF minfBox;
              MP4::DINF dinfBox;
                MP4::DREF drefBox;/// \todo fix constructor dref in lib
                  MP4::URN urnBox;
                  urnBox.setName("Name Here");
                  urnBox.setLocation("Location Here");
                  drefBox.setDataEntry(urnBox,0);
                dinfBox.setContent(drefBox,0);
              minfBox.setContent(dinfBox,0);

              MP4::STBL stblBox;
                MP4::STSD stsdBox;
                  if (tmpStr == "video"){//boxname = codec
                    MP4::VisualSampleEntry vse;
                    std::string tmpStr2 = it->second["codec"];
                    if (tmpStr2 == "H264"){
                      vse.setCodec("avc1");
                    }
                    vse.setWidth(it->second["width"].asInt());
                    vse.setHeight(it->second["height"].asInt());
                    stsdBox.setEntry(vse,0);
                  }else if(tmpStr == "audio"){//boxname = codec
                    MP4::AudioSampleEntry ase;
                    std::string tmpStr2 = it->second["codec"];
                    if (tmpStr2 == "AAC"){
                      ase.setCodec("mp4a");
                    }
                    ase.setSampleRate(it->second["rate"].asInt());
                    ase.setChannelCount(it->second["channels"].asInt());
                    ase.setSampleSize(it->second["length"].asInt());
                    stsdBox.setEntry(ase,0);
                  }
                stblBox.setContent(stsdBox,0);

                MP4::STTS sttsBox;
                for (int i = 0; i < it->second["keys"].size(); i++){
                  MP4::STTSEntry newEntry;
                  newEntry.sampleCount = it->second["keys"][i]["parts"].size();
                  newEntry.sampleDelta = it->second["keys"][i]["len"].asInt();
                  sttsBox.setSTTSEntry(newEntry, i);
                }
                stblBox.setContent(sttsBox,1);

                MP4::STSC stscBox;//probably wrong
                for (int i = 0; i < it->second["keys"].size(); i++){
                  MP4::STSCEntry newEntry;
                  newEntry.firstChunk = it->second["keys"][i]["num"].asInt();
                  newEntry.samplesPerChunk = it->second["keys"][i]["parts"].size();
                  newEntry.sampleDescriptionIndex = 1;
                  stscBox.setSTSCEntry(newEntry, i);
                }
                stblBox.setContent(stscBox,2);

                /*MP4::STSZ stszBox;
                /// \todo calculate byte position of DTSCkeyframes in MP4Sample
                // in it->second["keys"]["parts"]
                stszBox.setSampleSize(0);
                for (int i = 0; i < it->second["keys"].size(); i++){
                  stszBox.setEntrySize(it->second["keys"][i]["size"], i);
                }
                stblBox.setContent(stszBox,3);*/

                MP4::STCO stcoBox;
                for (int i = 0; i < it->second["keys"].size(); i++){
                  stcoBox.setChunkOffset(it->second["keys"][i]["size"].asInt(), i);//in bytes in file
                }
                stblBox.setContent(stcoBox,3);
              minfBox.setContent(stblBox,1);
            mdiaBox.setContent(minfBox, 2);
          trakBox.setContent(mdiaBox, 1);
        moovBox.setContent(trakBox, boxOffset);
        boxOffset++;
      }
      //end arbitrary
      //initial offset lengte ftyp, length moov + 8

      //update all STCO
    std::cout << std::string(moovBox.asBox(),moovBox.boxedSize());

    //mdat box alot
    //video
    //while()
    //for(input.seekNext(); input.getJSON(); input.seekNext())
    //cout << input.getJSON["data"].asString()

    printf("%c%c%c%cmdat", 0x00,0x00,0x01,0x00);
    std::set<int> selector;
    for (JSON::ObjIter trackIt = input.getMeta()["tracks"].ObjBegin(); trackIt != input.getMeta()["tracks"].ObjEnd(); trackIt++){
      selector.insert(trackIt->second["trackid"].asInt());
    }
    input.selectTracks(selector);

    input.seek_time(0);

    input.seekNext();
    while (input.getJSON()){
      //blaat
      input.seekNext();
    }
    return 0;
  } //DTSC2MP4

} //Converter namespace

/// Entry point for DTSC2FLV, simply calls Converters::DTSC2FLV().
int main(int argc, char ** argv){
  Util::Config conf = Util::Config(argv[0], PACKAGE_VERSION);
  conf.addOption("filename", JSON::fromString("{\"arg_num\":1, \"arg\":\"string\", \"help\":\"Filename of the input file to convert.\"}"));
  conf.parseArgs(argc, argv);
  return Converters::DTSC2MP4(conf);
} //main
<commit_msg>Commit just before tweaking. Output is sensible, but not correct.<commit_after>/// \file dtsc2mp4.cpp
/// Contains the code that will transform any valid DTSC input into valid MP4s.

#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <cstdlib>
#include <cstdio>
#include <string.h>
#include <unistd.h>
#include <signal.h>
#include <mist/json.h>
#include <mist/dtsc.h> //DTSC support
#include <mist/mp4.h> //MP4 support
#include <mist/ts_packet.h> //TS support ff
#include <mist/config.h>

///\brief Holds everything unique to converters.
namespace Converters {

  ///\brief Converts DTSC from file to MP4 on stdout.
  ///\return The return code for the converter.
  int DTSC2MP4(Util::Config & conf){
    DTSC::File input(conf.getString("filename"));
    //ftyp box
    /// \todo fill ftyp with non hardcoded values from file
    /*MP4::FTYP ftypBox;
    ftypBox.setMajorBrand(0x69736f6d);
    ftypBox.setMinorVersion(512);
    ftypBox.setCompatibleBrands(0x69736f6d,0);
    ftypBox.setCompatibleBrands(0x69736f32,1);
    ftypBox.setCompatibleBrands(0x61766331,2);
    ftypBox.setCompatibleBrands(0x6d703431,3);
    std::cout << std::string(ftypBox.asBox(),ftypBox.boxedSize());*/
    MP4::FTYP ftypBox;
    ftypBox.setMajorBrand(0x6D703431);//mp41
    ftypBox.setMinorVersion(0);
    ftypBox.setCompatibleBrands(0x6D703431,0);
    std::cout << std::string(ftypBox.asBox(),ftypBox.boxedSize());

    //moov box
    MP4::MOOV moovBox;
      MP4::MVHD mvhdBox;
      mvhdBox.setCreationTime(0);
      mvhdBox.setModificationTime(0);
      mvhdBox.setTimeScale(1000);
      mvhdBox.setDuration(input.getMeta()["lastms"].asInt());
      /// \todo mvhd setTrackID automatic fix (next track ID)
      moovBox.setContent(mvhdBox, 0);

      //start arbitrary track addition
      int boxOffset = 1;
      for (JSON::ObjIter it = input.getMeta()["tracks"].ObjBegin(); it != input.getMeta()["tracks"].ObjEnd(); it++){
        MP4::TRAK trakBox;
          MP4::TKHD tkhdBox;
          //std::cerr << it->second["trackid"].asInt() << std::endl;
          tkhdBox.setTrackID(it->second["trackid"].asInt());

          if (it->second["type"].asString() == "video"){
            tkhdBox.setWidth(it->second["width"].asInt() << 16);
            tkhdBox.setHeight(it->second["height"].asInt() << 16);
          }
          trakBox.setContent(tkhdBox, 0);

          MP4::MDIA mdiaBox;
            MP4::MDHD mdhdBox;/// \todo fix constructor mdhd in lib
            mdhdBox.setCreationTime(0);
            mdhdBox.setModificationTime(0);
            mdhdBox.setTimeScale(1000);
            mdhdBox.setDuration(input.getMeta()["lastms"].asInt());
            mdiaBox.setContent(mdhdBox, 0);

            std::string tmpStr = it->second["type"].asString();
            MP4::HDLR hdlrBox;/// \todo fix constructor hdlr in lib
            if (tmpStr == "video"){
              hdlrBox.setHandlerType(0x76696465);//vide
            }else if (tmpStr == "audio"){
              hdlrBox.setHandlerType(0x736F756E);//soun
            }
            hdlrBox.setName(it->first);
            mdiaBox.setContent(hdlrBox, 1);

            MP4::MINF minfBox;
              MP4::DINF dinfBox;
                MP4::DREF drefBox;/// \todo fix constructor dref in lib
                  MP4::URN urnBox;
                  urnBox.setName("Name Here");
                  urnBox.setLocation("Location Here");
                  drefBox.setDataEntry(urnBox,0);
                dinfBox.setContent(drefBox,0);
              minfBox.setContent(dinfBox,0);

              MP4::STBL stblBox;
                MP4::STSD stsdBox;
                  if (tmpStr == "video"){//boxname = codec
                    MP4::VisualSampleEntry vse;
                    std::string tmpStr2 = it->second["codec"];
                    if (tmpStr2 == "H264"){
                      vse.setCodec("avc1");

                    }
                    vse.setWidth(it->second["width"].asInt());
                    vse.setHeight(it->second["height"].asInt());
                    stsdBox.setEntry(vse,0);
                  }else if(tmpStr == "audio"){//boxname = codec
                    MP4::AudioSampleEntry ase;
                    std::string tmpStr2 = it->second["codec"];
                    if (tmpStr2 == "AAC"){
                      ase.setCodec("mp4a");
                    }
                    ase.setSampleRate(it->second["rate"].asInt());
                    ase.setChannelCount(it->second["channels"].asInt());
                    ase.setSampleSize(it->second["length"].asInt());
                    stsdBox.setEntry(ase,0);
                  }
                stblBox.setContent(stsdBox,0);

                MP4::STTS sttsBox;
                for (int i = 0; i < it->second["keys"].size(); i++){
                  MP4::STTSEntry newEntry;
                  newEntry.sampleCount = it->second["keys"][i]["parts"].size();
                  newEntry.sampleDelta = it->second["keys"][i]["len"].asInt();
                  sttsBox.setSTTSEntry(newEntry, i);
                }
                stblBox.setContent(sttsBox,1);

                MP4::STSC stscBox;//probably wrong
                for (int i = 0; i < it->second["keys"].size(); i++){
                  MP4::STSCEntry newEntry;
                  newEntry.firstChunk = it->second["keys"][i]["num"].asInt();
                  newEntry.samplesPerChunk = it->second["keys"][i]["parts"].size();
                  newEntry.sampleDescriptionIndex = 1;
                  stscBox.setSTSCEntry(newEntry, i);
                }
                stblBox.setContent(stscBox,2);

                /*MP4::STSZ stszBox;
                /// \todo calculate byte position of DTSCkeyframes in MP4Sample
                // in it->second["keys"]["parts"]
                stszBox.setSampleSize(0);
                for (int i = 0; i < it->second["keys"].size(); i++){
                  stszBox.setEntrySize(it->second["keys"][i]["size"], i);
                }
                stblBox.setContent(stszBox,3);*/
                MP4::STSZ stszBox;
                for (int i = 0; i < it->second["keys"].size(); i++){
                  stszBox.setEntrySize(it->second["keys"][i]["size"].asInt(), i);//in bytes in file
                }
                stblBox.setContent(stszBox,3);

                MP4::STCO stcoBox;
                for (int i = 0; i < it->second["keys"].size(); i++){
                  stcoBox.setChunkOffset(it->second["keys"][i]["size"].asInt(), i);//in bytes in file
                }
                stblBox.setContent(stcoBox,4);
              minfBox.setContent(stblBox,1);
            mdiaBox.setContent(minfBox, 2);
          trakBox.setContent(mdiaBox, 1);
        moovBox.setContent(trakBox, boxOffset);
        boxOffset++;
      }
      //end arbitrary
      //initial offset lengte ftyp, length moov + 8
      unsigned long long int byteOffset = ftypBox.boxedSize() + moovBox.boxedSize() + 8;
      //update all STCO
      //for tracks
      for (unsigned int i = 1; i < moovBox.getContentCount(); i++){
        //10 lines to get the STCO box.
        MP4::TRAK checkTrakBox;
        MP4::MDIA checkMdiaBox;
        MP4::MINF checkMinfBox;
        MP4::STBL checkStblBox;
        MP4::STCO checkStcoBox;
        checkTrakBox = ((MP4::TRAK&)moovBox.getContent(i));
        checkMdiaBox = ((MP4::MDIA&)checkTrakBox.getContent(1));
        checkMinfBox = ((MP4::MINF&)checkMdiaBox.getContent(2));
        checkStblBox = ((MP4::STBL&)checkMinfBox.getContent(1));
        checkStcoBox = ((MP4::STCO&)checkStblBox.getContent(4));

        //std::cerr << std::string(checkStcoBox.asBox(),checkStcoBox.boxedSize()) << std::endl;
        for (unsigned int o = 0; o < checkStcoBox.getEntryCount(); o++){
          uint64_t temp;
          temp = checkStcoBox.getChunkOffset(o);
          checkStcoBox.setChunkOffset(byteOffset, o);
          byteOffset += temp;
          //std::cerr << "FNURF "<< byteOffset << ", " << temp << std::endl;
        }
      }
    std::cout << std::string(moovBox.asBox(),moovBox.boxedSize());

    //mdat box alot
    //video
    //while()
    //for(input.seekNext(); input.getJSON(); input.seekNext())
    //cout << input.getJSON["data"].asString()

    std::set<int> selector;
    for (JSON::ObjIter trackIt = input.getMeta()["tracks"].ObjBegin(); trackIt != input.getMeta()["tracks"].ObjEnd(); trackIt++){
      selector.insert(trackIt->second["trackid"].asInt());
    }
    input.selectTracks(selector);

    input.seek_time(0);

    input.seekNext();
    std::vector<std::string> dataParts;
    while (input.getJSON()){
      //if not in vector, create;
      if (dataParts.size() < input.getJSON()["trackid"].asInt()){
        dataParts.resize(input.getJSON()["trackid"].asInt());
      }
      //putting everything in its place
      dataParts[input.getJSON()["trackid"].asInt()-1] += input.getJSON()["data"].asString();
      input.seekNext();
    }
    uint32_t mdatSize = 0;
    for (unsigned int x = 0; x < dataParts.size(); x++){
      mdatSize += dataParts[x].size();
    }
    //std::cerr << "Total Data size: " << mdatSize << std::endl;
    printf("%c%c%c%cmdat", (mdatSize>>24) & 0x000000FF,(mdatSize>>16) & 0x000000FF,(mdatSize>>8) & 0x000000FF,mdatSize & 0x000000FF);
    for (unsigned int x = 0; x < dataParts.size(); x++){
      std::cout << dataParts[x];
    }
    return 0;
  } //DTSC2MP4

} //Converter namespace

/// Entry point for DTSC2FLV, simply calls Converters::DTSC2FLV().
int main(int argc, char ** argv){
  Util::Config conf = Util::Config(argv[0], PACKAGE_VERSION);
  conf.addOption("filename", JSON::fromString("{\"arg_num\":1, \"arg\":\"string\", \"help\":\"Filename of the input file to convert.\"}"));
  conf.parseArgs(argc, argv);
  return Converters::DTSC2MP4(conf);
} //main
<|endoftext|>