<commit_before>// Copyright 2008, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

//
// NPAPI UnitTests.
//

// windows headers
#include <windows.h>
#include <shellapi.h>
#include <shlobj.h>
#include <atlbase.h>
#include <comutil.h>

// runtime headers
#include <stdlib.h>
#include <string.h>
#include <memory.h>

#include <ostream>

#include "base/file_util.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/test/automation/tab_proxy.h"
#include "chrome/test/ui/ui_test.h"
#include "net/base/net_util.h"

const char kTestCompleteCookie[] = "status";
const char kTestCompleteSuccess[] = "OK";
const int kLongWaitTimeout = 30*1000;
const int kShortWaitTimeout = 5*1000;

std::ostream& operator<<(std::ostream& out, const CComBSTR &str)
{
  // I love strings.  I really do.  That's why I make sure
  // to need 4 different types of strings to stream one out.
  TCHAR szFinal[1024];
  _bstr_t bstrIntermediate(str);
  _stprintf_s(szFinal, _T("%s"), (LPCTSTR)bstrIntermediate);
  return out << szFinal;
}

class NPAPITester : public UITest {
 protected:
  NPAPITester() : UITest()
  {
  }

  virtual void SetUp()
  {
    // We need to copy our test-plugin into the plugins directory so that
    // the browser can load it.
    std::wstring plugins_directory = browser_directory_ + L"\\plugins";
    std::wstring plugin_src = browser_directory_ + L"\\npapi_test_plugin.dll";
    plugin_dll_ = plugins_directory + L"\\npapi_test_plugin.dll";

    CreateDirectory(plugins_directory.c_str(), NULL);
    CopyFile(plugin_src.c_str(), plugin_dll_.c_str(), FALSE);

    UITest::SetUp();
  }

  virtual void TearDown()
  {
    DeleteFile(plugin_dll_.c_str());
    UITest::TearDown();
  }

protected:
  // Generate the URL for testing a particular test.
  // HTML for the tests is all located in test_directory\npapi\<testcase>
  GURL GetTestUrl(const std::wstring &test_case) {
    std::wstring path;
    PathService::Get(chrome::DIR_TEST_DATA, &path);
    file_util::AppendToPath(&path, L"npapi");
    file_util::AppendToPath(&path, test_case);
    return net::FilePathToFileURL(path);
  }

  // Waits for the test case to finish.
  // ASSERTS if there are test failures.
  void WaitForFinish(const std::string &name, const std::string &id,
                     const GURL &url, const int wait_time)
  {
    const int kSleepTime = 250;      // 4 times per second
    const int kMaxIntervals = wait_time / kSleepTime;

    scoped_ptr<TabProxy> tab(GetActiveTab());

    std::string done_str;
    for (int i = 0; i < kMaxIntervals; ++i) {
      Sleep(kSleepTime);

      // The webpage being tested has javascript which sets a cookie
      // which signals completion of the test.  The cookie name is
      // a concatenation of the test name and the test id.  This allows
      // us to run multiple tests within a single webpage and test
      // that they all c
      std::string cookieName = name;
      cookieName.append(".");
      cookieName.append(id);
      cookieName.append(".");
      cookieName.append(kTestCompleteCookie);
      tab->GetCookieByName(url, cookieName, &done_str);
      if (!done_str.empty())
        break;
    }

    EXPECT_EQ(kTestCompleteSuccess, done_str);
  }

private:
  std::wstring plugin_dll_;
};

class NPAPIVisiblePluginTester : public NPAPITester {
  protected:
  virtual void SetUp() {
    show_window_ = true;
    NPAPITester::SetUp();
  }
};

// Test passing arguments to a plugin.
TEST_F(NPAPITester, Arguments) {
  std::wstring test_case = L"arguments.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("arguments", "1", url, kShortWaitTimeout);
}

// Test invoking many plugins within a single page.
TEST_F(NPAPITester, ManyPlugins) {
  std::wstring test_case = L"many_plugins.html";
  GURL url(GetTestUrl(test_case));
  NavigateToURL(url);
  WaitForFinish("arguments", "1", url, kShortWaitTimeout);
  WaitForFinish("arguments", "2", url, kShortWaitTimeout);
  WaitForFinish("arguments", "3", url, kShortWaitTimeout);
  WaitForFinish("arguments", "4", url, kShortWaitTimeout);
  WaitForFinish("arguments", "5", url, kShortWaitTimeout);
  WaitForFinish("arguments", "6", url, kShortWaitTimeout);
  WaitForFinish("arguments", "7", url, kShortWaitTimeout);
  WaitForFinish("arguments", "8", url, kShortWaitTimeout);
  WaitForFinish("arguments", "9", url, kShortWaitTimeout);
  WaitForFinish("arguments", "10", url, kShortWaitTimeout);
  WaitForFinish("arguments", "11", url, kShortWaitTimeout);
  WaitForFinish("arguments", "12", url, kShortWaitTimeout);
  WaitForFinish("arguments", "13", url, kShortWaitTimeout);
  WaitForFinish("arguments", "14", url, kShortWaitTimeout);
  WaitForFinish("arguments", "15", url, kShortWaitTimeout);
}

// Test various calls to GetURL from a plugin.
TEST_F(NPAPITester, GetURL) {
  std::wstring test_case = L"geturl.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("geturl", "1", url, kShortWaitTimeout);
}

// Test various calls to GetURL for javascript URLs with
// non NULL targets from a plugin.
TEST_F(NPAPITester, GetJavaScriptURL) {
  std::wstring test_case = L"get_javascript_url.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("getjavascripturl", "1", url, kShortWaitTimeout);
}

// Tests that if an NPObject is proxies back to its original process, the
// original pointer is returned and not a proxy.  If this fails the plugin
// will crash.
TEST_F(NPAPITester, NPObjectProxy) {
  std::wstring test_case = L"npobject_proxy.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("npobject_proxy", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script using NPN_GetURL
// works without crashing or hanging
TEST_F(NPAPITester, SelfDeletePluginGetUrl) {
  std::wstring test_case = L"self_delete_plugin_geturl.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("self_delete_plugin_geturl", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script using Invoke
// works without crashing or hanging
TEST_F(NPAPITester, SelfDeletePluginInvoke) {
  std::wstring test_case = L"self_delete_plugin_invoke.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("self_delete_plugin_invoke", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script in the context of
// a synchronous paint event works correctly
TEST_F(NPAPIVisiblePluginTester, SelfDeletePluginInvokeInSynchronousPaint) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"execute_script_delete_in_paint.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("execute_script_delete_in_paint", "1", url, kShortWaitTimeout);
  }
}

TEST_F(NPAPIVisiblePluginTester, SelfDeletePluginInNewStream) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"self_delete_plugin_stream.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("self_delete_plugin_stream", "1", url, kShortWaitTimeout);
  }
}

// Tests if a plugin has a non zero window rect.
TEST_F(NPAPIVisiblePluginTester, VerifyPluginWindowRect) {
  show_window_ = true;
  std::wstring test_case = L"verify_plugin_window_rect.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("checkwindowrect", "1", url, kShortWaitTimeout);
}

TEST_F(NPAPIVisiblePluginTester, VerifyNPObjectLifetimeTest) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"npobject_lifetime_test.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("npobject_lifetime_test", "1", url, kShortWaitTimeout);
  }
}

// Tests that we don't crash or assert if NPP_New fails
TEST_F(NPAPIVisiblePluginTester, NewFails) {
  GURL url = GetTestUrl(L"new_fails.html");
  NavigateToURL(url);
  WaitForFinish("new_fails", "1", url, kShortWaitTimeout);
}

TEST_F(NPAPIVisiblePluginTester, SelfDeletePluginInNPNEvaluate) {
 if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    GURL url = GetTestUrl(L"execute_script_delete_in_npn_evaluate.html");
    NavigateToURL(url);
    WaitForFinish("npobject_delete_plugin_in_evaluate", "1", url,
                  kShortWaitTimeout);
 }
}<commit_msg>Disable tests while I investigate<commit_after>// Copyright 2008, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

//
// NPAPI UnitTests.
//

// windows headers
#include <windows.h>
#include <shellapi.h>
#include <shlobj.h>
#include <atlbase.h>
#include <comutil.h>

// runtime headers
#include <stdlib.h>
#include <string.h>
#include <memory.h>

#include <ostream>

#include "base/file_util.h"
#include "chrome/common/chrome_paths.h"
#include "chrome/test/automation/tab_proxy.h"
#include "chrome/test/ui/ui_test.h"
#include "net/base/net_util.h"

const char kTestCompleteCookie[] = "status";
const char kTestCompleteSuccess[] = "OK";
const int kLongWaitTimeout = 30*1000;
const int kShortWaitTimeout = 5*1000;

std::ostream& operator<<(std::ostream& out, const CComBSTR &str)
{
  // I love strings.  I really do.  That's why I make sure
  // to need 4 different types of strings to stream one out.
  TCHAR szFinal[1024];
  _bstr_t bstrIntermediate(str);
  _stprintf_s(szFinal, _T("%s"), (LPCTSTR)bstrIntermediate);
  return out << szFinal;
}

class NPAPITester : public UITest {
 protected:
  NPAPITester() : UITest()
  {
  }

  virtual void SetUp()
  {
    // We need to copy our test-plugin into the plugins directory so that
    // the browser can load it.
    std::wstring plugins_directory = browser_directory_ + L"\\plugins";
    std::wstring plugin_src = browser_directory_ + L"\\npapi_test_plugin.dll";
    plugin_dll_ = plugins_directory + L"\\npapi_test_plugin.dll";

    CreateDirectory(plugins_directory.c_str(), NULL);
    CopyFile(plugin_src.c_str(), plugin_dll_.c_str(), FALSE);

    UITest::SetUp();
  }

  virtual void TearDown()
  {
    DeleteFile(plugin_dll_.c_str());
    UITest::TearDown();
  }

protected:
  // Generate the URL for testing a particular test.
  // HTML for the tests is all located in test_directory\npapi\<testcase>
  GURL GetTestUrl(const std::wstring &test_case) {
    std::wstring path;
    PathService::Get(chrome::DIR_TEST_DATA, &path);
    file_util::AppendToPath(&path, L"npapi");
    file_util::AppendToPath(&path, test_case);
    return net::FilePathToFileURL(path);
  }

  // Waits for the test case to finish.
  // ASSERTS if there are test failures.
  void WaitForFinish(const std::string &name, const std::string &id,
                     const GURL &url, const int wait_time)
  {
    const int kSleepTime = 250;      // 4 times per second
    const int kMaxIntervals = wait_time / kSleepTime;

    scoped_ptr<TabProxy> tab(GetActiveTab());

    std::string done_str;
    for (int i = 0; i < kMaxIntervals; ++i) {
      Sleep(kSleepTime);

      // The webpage being tested has javascript which sets a cookie
      // which signals completion of the test.  The cookie name is
      // a concatenation of the test name and the test id.  This allows
      // us to run multiple tests within a single webpage and test
      // that they all c
      std::string cookieName = name;
      cookieName.append(".");
      cookieName.append(id);
      cookieName.append(".");
      cookieName.append(kTestCompleteCookie);
      tab->GetCookieByName(url, cookieName, &done_str);
      if (!done_str.empty())
        break;
    }

    EXPECT_EQ(kTestCompleteSuccess, done_str);
  }

private:
  std::wstring plugin_dll_;
};

class NPAPIVisiblePluginTester : public NPAPITester {
  protected:
  virtual void SetUp() {
    show_window_ = true;
    NPAPITester::SetUp();
  }
};

// Test passing arguments to a plugin.
TEST_F(NPAPITester, Arguments) {
  std::wstring test_case = L"arguments.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("arguments", "1", url, kShortWaitTimeout);
}

// Test invoking many plugins within a single page.
TEST_F(NPAPITester, ManyPlugins) {
  std::wstring test_case = L"many_plugins.html";
  GURL url(GetTestUrl(test_case));
  NavigateToURL(url);
  WaitForFinish("arguments", "1", url, kShortWaitTimeout);
  WaitForFinish("arguments", "2", url, kShortWaitTimeout);
  WaitForFinish("arguments", "3", url, kShortWaitTimeout);
  WaitForFinish("arguments", "4", url, kShortWaitTimeout);
  WaitForFinish("arguments", "5", url, kShortWaitTimeout);
  WaitForFinish("arguments", "6", url, kShortWaitTimeout);
  WaitForFinish("arguments", "7", url, kShortWaitTimeout);
  WaitForFinish("arguments", "8", url, kShortWaitTimeout);
  WaitForFinish("arguments", "9", url, kShortWaitTimeout);
  WaitForFinish("arguments", "10", url, kShortWaitTimeout);
  WaitForFinish("arguments", "11", url, kShortWaitTimeout);
  WaitForFinish("arguments", "12", url, kShortWaitTimeout);
  WaitForFinish("arguments", "13", url, kShortWaitTimeout);
  WaitForFinish("arguments", "14", url, kShortWaitTimeout);
  WaitForFinish("arguments", "15", url, kShortWaitTimeout);
}

// Test various calls to GetURL from a plugin.
TEST_F(NPAPITester, GetURL) {
  std::wstring test_case = L"geturl.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("geturl", "1", url, kShortWaitTimeout);
}

// Test various calls to GetURL for javascript URLs with
// non NULL targets from a plugin.
TEST_F(NPAPITester, GetJavaScriptURL) {
  std::wstring test_case = L"get_javascript_url.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("getjavascripturl", "1", url, kShortWaitTimeout);
}

// Tests that if an NPObject is proxies back to its original process, the
// original pointer is returned and not a proxy.  If this fails the plugin
// will crash.
TEST_F(NPAPITester, NPObjectProxy) {
  std::wstring test_case = L"npobject_proxy.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("npobject_proxy", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script using NPN_GetURL
// works without crashing or hanging
TEST_F(NPAPITester, SelfDeletePluginGetUrl) {
  std::wstring test_case = L"self_delete_plugin_geturl.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("self_delete_plugin_geturl", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script using Invoke
// works without crashing or hanging
TEST_F(NPAPITester, SelfDeletePluginInvoke) {
  std::wstring test_case = L"self_delete_plugin_invoke.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("self_delete_plugin_invoke", "1", url, kShortWaitTimeout);
}

// Tests if a plugin executing a self deleting script in the context of
// a synchronous paint event works correctly
TEST_F(NPAPIVisiblePluginTester, DISABLED_SelfDeletePluginInvokeInSynchronousPaint) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"execute_script_delete_in_paint.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("execute_script_delete_in_paint", "1", url, kShortWaitTimeout);
  }
}

TEST_F(NPAPIVisiblePluginTester, SelfDeletePluginInNewStream) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"self_delete_plugin_stream.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("self_delete_plugin_stream", "1", url, kShortWaitTimeout);
  }
}

// Tests if a plugin has a non zero window rect.
TEST_F(NPAPIVisiblePluginTester, DISABLED_VerifyPluginWindowRect) {
  show_window_ = true;
  std::wstring test_case = L"verify_plugin_window_rect.html";
  GURL url = GetTestUrl(test_case);
  NavigateToURL(url);
  WaitForFinish("checkwindowrect", "1", url, kShortWaitTimeout);
}

TEST_F(NPAPIVisiblePluginTester, VerifyNPObjectLifetimeTest) {
  if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    show_window_ = true;
    std::wstring test_case = L"npobject_lifetime_test.html";
    GURL url = GetTestUrl(test_case);
    NavigateToURL(url);
    WaitForFinish("npobject_lifetime_test", "1", url, kShortWaitTimeout);
  }
}

// Tests that we don't crash or assert if NPP_New fails
TEST_F(NPAPIVisiblePluginTester, NewFails) {
  GURL url = GetTestUrl(L"new_fails.html");
  NavigateToURL(url);
  WaitForFinish("new_fails", "1", url, kShortWaitTimeout);
}

TEST_F(NPAPIVisiblePluginTester, SelfDeletePluginInNPNEvaluate) {
 if (!UITest::in_process_plugins() && !UITest::in_process_renderer()) {
    GURL url = GetTestUrl(L"execute_script_delete_in_npn_evaluate.html");
    NavigateToURL(url);
    WaitForFinish("npobject_delete_plugin_in_evaluate", "1", url,
                  kShortWaitTimeout);
 }
}<|endoftext|>


<commit_before>// Copyright (c) 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chromeos/network/cert_loader.h"

#include <algorithm>

#include "base/chromeos/chromeos_version.h"
#include "base/observer_list.h"
#include "base/strings/string_number_conversions.h"
#include "base/task_runner_util.h"
#include "base/threading/worker_pool.h"
#include "chromeos/dbus/cryptohome_client.h"
#include "chromeos/dbus/dbus_thread_manager.h"
#include "crypto/encryptor.h"
#include "crypto/nss_util.h"
#include "crypto/sha2.h"
#include "crypto/symmetric_key.h"
#include "net/cert/nss_cert_database.h"

namespace chromeos {

namespace {

const int64 kInitialRequestDelayMs = 100;
const int64 kMaxRequestDelayMs = 300000; // 5 minutes

// Calculates the delay before running next attempt to initiatialize the TPM
// token, if |last_delay| was the last or initial delay.
base::TimeDelta GetNextRequestDelayMs(base::TimeDelta last_delay) {
  // This implements an exponential backoff, as we don't know in which order of
  // magnitude the TPM token changes it's state.
  base::TimeDelta next_delay = last_delay * 2;

  // Cap the delay to prevent an overflow. This threshold is arbitrarily chosen.
  const base::TimeDelta max_delay =
      base::TimeDelta::FromMilliseconds(kMaxRequestDelayMs);
  if (next_delay > max_delay)
    next_delay = max_delay;
  return next_delay;
}

void LoadNSSCertificates(net::CertificateList* cert_list) {
  if (base::chromeos::IsRunningOnChromeOS())
    net::NSSCertDatabase::GetInstance()->ListCerts(cert_list);
}

}  // namespace

static CertLoader* g_cert_loader = NULL;

// static
void CertLoader::Initialize() {
  CHECK(!g_cert_loader);
  g_cert_loader = new CertLoader();
}

// static
void CertLoader::Shutdown() {
  CHECK(g_cert_loader);
  delete g_cert_loader;
  g_cert_loader = NULL;
}

// static
CertLoader* CertLoader::Get() {
  CHECK(g_cert_loader) << "CertLoader::Get() called before Initialize()";
  return g_cert_loader;
}

// static
bool CertLoader::IsInitialized() {
  return g_cert_loader;
}

CertLoader::CertLoader()
    : certificates_requested_(false),
      certificates_loaded_(false),
      certificates_update_required_(false),
      certificates_update_running_(false),
      tpm_token_state_(TPM_STATE_UNKNOWN),
      tpm_request_delay_(
          base::TimeDelta::FromMilliseconds(kInitialRequestDelayMs)),
      initialize_token_factory_(this),
      update_certificates_factory_(this) {
  net::CertDatabase::GetInstance()->AddObserver(this);
  LoginState::Get()->AddObserver(this);
  RequestCertificates();
}

CertLoader::~CertLoader() {
  net::CertDatabase::GetInstance()->RemoveObserver(this);
  LoginState::Get()->RemoveObserver(this);
}

void CertLoader::AddObserver(CertLoader::Observer* observer) {
  observers_.AddObserver(observer);
}

void CertLoader::RemoveObserver(CertLoader::Observer* observer) {
  observers_.RemoveObserver(observer);
}

bool CertLoader::CertificatesLoading() const {
  return certificates_requested_ && !certificates_loaded_;
}

bool CertLoader::IsHardwareBacked() const {
  return !tpm_token_name_.empty();
}

void CertLoader::RequestCertificates() {
  CHECK(thread_checker_.CalledOnValidThread());
  VLOG(1) << "RequestCertificates: " << LoginState::Get()->IsUserLoggedIn();
  if (certificates_requested_ || !LoginState::Get()->IsUserLoggedIn())
    return;

  certificates_requested_ = true;

  // Ensure we've opened the user's key/certificate database.
  crypto::OpenPersistentNSSDB();
  if (base::chromeos::IsRunningOnChromeOS())
    crypto::EnableTPMTokenForNSS();

  // This is the entry point to the TPM token initialization process, which we
  // should do at most once.
  DCHECK(!initialize_token_factory_.HasWeakPtrs());
  InitializeTokenAndLoadCertificates();
}

void CertLoader::InitializeTokenAndLoadCertificates() {
  CHECK(thread_checker_.CalledOnValidThread());
  VLOG(1) << "InitializeTokenAndLoadCertificates";

  switch(tpm_token_state_) {
    case TPM_STATE_UNKNOWN: {
      DBusThreadManager::Get()->GetCryptohomeClient()->TpmIsEnabled(
      base::Bind(&CertLoader::OnTpmIsEnabled,
                 initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_DISABLED: {
      // TPM is disabled, so proceed with empty tpm token name.
      StartLoadCertificates();
      return;
    }
    case TPM_ENABLED: {
      DBusThreadManager::Get()->GetCryptohomeClient()->Pkcs11IsTpmTokenReady(
          base::Bind(&CertLoader::OnPkcs11IsTpmTokenReady,
                     initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_TOKEN_READY: {
      // Retrieve token_name_ and user_pin_ here since they will never change
      // and CryptohomeClient calls are not thread safe.
      DBusThreadManager::Get()->GetCryptohomeClient()->Pkcs11GetTpmTokenInfo(
          base::Bind(&CertLoader::OnPkcs11GetTpmTokenInfo,
                     initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_TOKEN_INFO_RECEIVED: {
      InitializeNSSForTPMToken();
      return;
    }
    case TPM_TOKEN_NSS_INITIALIZED: {
      StartLoadCertificates();
      return;
    }
  }
}

void CertLoader::RetryTokenInitializationLater() {
  LOG(WARNING) << "Re-Requesting Certificates later.";
  MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&CertLoader::InitializeTokenAndLoadCertificates,
                 initialize_token_factory_.GetWeakPtr()),
      tpm_request_delay_);
  tpm_request_delay_ = GetNextRequestDelayMs(tpm_request_delay_);
}

// For background see this discussion on dev-tech-crypto.lists.mozilla.org:
// http://web.archiveorange.com/archive/v/6JJW7E40sypfZGtbkzxX
//
// NOTE: This function relies on the convention that the same PKCS#11 ID
// is shared between a certificate and its associated private and public
// keys.  I tried to implement this with PK11_GetLowLevelKeyIDForCert(),
// but that always returns NULL on Chrome OS for me.
std::string CertLoader::GetPkcs11IdForCert(
    const net::X509Certificate& cert) const {
  if (!IsHardwareBacked())
    return std::string();

  CERTCertificateStr* cert_handle = cert.os_cert_handle();
  SECKEYPrivateKey *priv_key =
      PK11_FindKeyByAnyCert(cert_handle, NULL /* wincx */);
  if (!priv_key)
    return std::string();

  // Get the CKA_ID attribute for a key.
  SECItem* sec_item = PK11_GetLowLevelKeyIDForPrivateKey(priv_key);
  std::string pkcs11_id;
  if (sec_item) {
    pkcs11_id = base::HexEncode(sec_item->data, sec_item->len);
    SECITEM_FreeItem(sec_item, PR_TRUE);
  }
  SECKEY_DestroyPrivateKey(priv_key);

  return pkcs11_id;
}

void CertLoader::OnTpmIsEnabled(DBusMethodCallStatus call_status,
                                bool tpm_is_enabled) {
  VLOG(1) << "OnTpmIsEnabled: " << tpm_is_enabled;

  if (call_status == DBUS_METHOD_CALL_SUCCESS && tpm_is_enabled)
    tpm_token_state_ = TPM_ENABLED;
  else
    tpm_token_state_ = TPM_DISABLED;

  InitializeTokenAndLoadCertificates();
}

void CertLoader::OnPkcs11IsTpmTokenReady(DBusMethodCallStatus call_status,
                                         bool is_tpm_token_ready) {
  VLOG(1) << "OnPkcs11IsTpmTokenReady: " << is_tpm_token_ready;

  if (call_status == DBUS_METHOD_CALL_FAILURE || !is_tpm_token_ready) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_state_ = TPM_TOKEN_READY;
  InitializeTokenAndLoadCertificates();
}

void CertLoader::OnPkcs11GetTpmTokenInfo(DBusMethodCallStatus call_status,
                                         const std::string& token_name,
                                         const std::string& user_pin) {
  VLOG(1) << "OnPkcs11GetTpmTokenInfo: " << token_name;

  if (call_status == DBUS_METHOD_CALL_FAILURE) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_name_ = token_name;
  // TODO(stevenjb): The network code expects a slot ID, not a label. See
  // crbug.com/201101. For now, use a hard coded, well known slot instead.
  const char kHardcodedTpmSlot[] = "0";
  tpm_token_slot_ = kHardcodedTpmSlot;
  tpm_user_pin_ = user_pin;
  tpm_token_state_ = TPM_TOKEN_INFO_RECEIVED;

  InitializeTokenAndLoadCertificates();
}

void CertLoader::InitializeNSSForTPMToken() {
  VLOG(1) << "InitializeNSSForTPMToken";

  if (base::chromeos::IsRunningOnChromeOS() &&
      !crypto::InitializeTPMToken(tpm_token_name_, tpm_user_pin_)) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_state_ = TPM_TOKEN_NSS_INITIALIZED;
  InitializeTokenAndLoadCertificates();
}

void CertLoader::StartLoadCertificates() {
  VLOG(1) << "StartLoadCertificates";

  if (certificates_update_running_) {
    certificates_update_required_ = true;
    return;
  }

  net::CertificateList* cert_list = new net::CertificateList;
  certificates_update_running_ = true;
  base::WorkerPool::GetTaskRunner(true /* task_is_slow */)->
      PostTaskAndReply(
          FROM_HERE,
          base::Bind(LoadNSSCertificates, cert_list),
          base::Bind(&CertLoader::UpdateCertificates,
                     update_certificates_factory_.GetWeakPtr(),
                     base::Owned(cert_list)));
}

void CertLoader::UpdateCertificates(net::CertificateList* cert_list) {
  CHECK(thread_checker_.CalledOnValidThread());
  DCHECK(certificates_update_running_);
  VLOG(1) << "UpdateCertificates: " << cert_list->size();

  // Ignore any existing certificates.
  cert_list_.swap(*cert_list);

  NotifyCertificatesLoaded(!certificates_loaded_);
  certificates_loaded_ = true;

  certificates_update_running_ = false;
  if (certificates_update_required_)
    StartLoadCertificates();
}

void CertLoader::NotifyCertificatesLoaded(bool initial_load) {
  FOR_EACH_OBSERVER(Observer, observers_,
                    OnCertificatesLoaded(cert_list_, initial_load));
}

void CertLoader::OnCertTrustChanged(const net::X509Certificate* cert) {
}

void CertLoader::OnCertAdded(const net::X509Certificate* cert) {
  VLOG(1) << "OnCertAdded";
  StartLoadCertificates();
}

void CertLoader::OnCertRemoved(const net::X509Certificate* cert) {
  VLOG(1) << "OnCertRemoved";
  StartLoadCertificates();
}

void CertLoader::LoggedInStateChanged(LoginState::LoggedInState state) {
  VLOG(1) << "LoggedInStateChanged: " << state;
  RequestCertificates();
}

}  // namespace chromeos
<commit_msg>Fix a busy loop in CertLoader.<commit_after>// Copyright (c) 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chromeos/network/cert_loader.h"

#include <algorithm>

#include "base/chromeos/chromeos_version.h"
#include "base/observer_list.h"
#include "base/strings/string_number_conversions.h"
#include "base/task_runner_util.h"
#include "base/threading/worker_pool.h"
#include "chromeos/dbus/cryptohome_client.h"
#include "chromeos/dbus/dbus_thread_manager.h"
#include "crypto/encryptor.h"
#include "crypto/nss_util.h"
#include "crypto/sha2.h"
#include "crypto/symmetric_key.h"
#include "net/cert/nss_cert_database.h"

namespace chromeos {

namespace {

const int64 kInitialRequestDelayMs = 100;
const int64 kMaxRequestDelayMs = 300000; // 5 minutes

// Calculates the delay before running next attempt to initiatialize the TPM
// token, if |last_delay| was the last or initial delay.
base::TimeDelta GetNextRequestDelayMs(base::TimeDelta last_delay) {
  // This implements an exponential backoff, as we don't know in which order of
  // magnitude the TPM token changes it's state.
  base::TimeDelta next_delay = last_delay * 2;

  // Cap the delay to prevent an overflow. This threshold is arbitrarily chosen.
  const base::TimeDelta max_delay =
      base::TimeDelta::FromMilliseconds(kMaxRequestDelayMs);
  if (next_delay > max_delay)
    next_delay = max_delay;
  return next_delay;
}

void LoadNSSCertificates(net::CertificateList* cert_list) {
  if (base::chromeos::IsRunningOnChromeOS())
    net::NSSCertDatabase::GetInstance()->ListCerts(cert_list);
}

}  // namespace

static CertLoader* g_cert_loader = NULL;

// static
void CertLoader::Initialize() {
  CHECK(!g_cert_loader);
  g_cert_loader = new CertLoader();
}

// static
void CertLoader::Shutdown() {
  CHECK(g_cert_loader);
  delete g_cert_loader;
  g_cert_loader = NULL;
}

// static
CertLoader* CertLoader::Get() {
  CHECK(g_cert_loader) << "CertLoader::Get() called before Initialize()";
  return g_cert_loader;
}

// static
bool CertLoader::IsInitialized() {
  return g_cert_loader;
}

CertLoader::CertLoader()
    : certificates_requested_(false),
      certificates_loaded_(false),
      certificates_update_required_(false),
      certificates_update_running_(false),
      tpm_token_state_(TPM_STATE_UNKNOWN),
      tpm_request_delay_(
          base::TimeDelta::FromMilliseconds(kInitialRequestDelayMs)),
      initialize_token_factory_(this),
      update_certificates_factory_(this) {
  net::CertDatabase::GetInstance()->AddObserver(this);
  LoginState::Get()->AddObserver(this);
  RequestCertificates();
}

CertLoader::~CertLoader() {
  net::CertDatabase::GetInstance()->RemoveObserver(this);
  LoginState::Get()->RemoveObserver(this);
}

void CertLoader::AddObserver(CertLoader::Observer* observer) {
  observers_.AddObserver(observer);
}

void CertLoader::RemoveObserver(CertLoader::Observer* observer) {
  observers_.RemoveObserver(observer);
}

bool CertLoader::CertificatesLoading() const {
  return certificates_requested_ && !certificates_loaded_;
}

bool CertLoader::IsHardwareBacked() const {
  return !tpm_token_name_.empty();
}

void CertLoader::RequestCertificates() {
  CHECK(thread_checker_.CalledOnValidThread());
  VLOG(1) << "RequestCertificates: " << LoginState::Get()->IsUserLoggedIn();
  if (certificates_requested_ || !LoginState::Get()->IsUserLoggedIn())
    return;

  certificates_requested_ = true;

  // Ensure we've opened the user's key/certificate database.
  crypto::OpenPersistentNSSDB();
  if (base::chromeos::IsRunningOnChromeOS())
    crypto::EnableTPMTokenForNSS();

  // This is the entry point to the TPM token initialization process, which we
  // should do at most once.
  DCHECK(!initialize_token_factory_.HasWeakPtrs());
  InitializeTokenAndLoadCertificates();
}

void CertLoader::InitializeTokenAndLoadCertificates() {
  CHECK(thread_checker_.CalledOnValidThread());
  VLOG(1) << "InitializeTokenAndLoadCertificates";

  switch(tpm_token_state_) {
    case TPM_STATE_UNKNOWN: {
      DBusThreadManager::Get()->GetCryptohomeClient()->TpmIsEnabled(
      base::Bind(&CertLoader::OnTpmIsEnabled,
                 initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_DISABLED: {
      // TPM is disabled, so proceed with empty tpm token name.
      StartLoadCertificates();
      return;
    }
    case TPM_ENABLED: {
      DBusThreadManager::Get()->GetCryptohomeClient()->Pkcs11IsTpmTokenReady(
          base::Bind(&CertLoader::OnPkcs11IsTpmTokenReady,
                     initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_TOKEN_READY: {
      // Retrieve token_name_ and user_pin_ here since they will never change
      // and CryptohomeClient calls are not thread safe.
      DBusThreadManager::Get()->GetCryptohomeClient()->Pkcs11GetTpmTokenInfo(
          base::Bind(&CertLoader::OnPkcs11GetTpmTokenInfo,
                     initialize_token_factory_.GetWeakPtr()));
      return;
    }
    case TPM_TOKEN_INFO_RECEIVED: {
      InitializeNSSForTPMToken();
      return;
    }
    case TPM_TOKEN_NSS_INITIALIZED: {
      StartLoadCertificates();
      return;
    }
  }
}

void CertLoader::RetryTokenInitializationLater() {
  LOG(WARNING) << "Re-Requesting Certificates later.";
  MessageLoop::current()->PostDelayedTask(
      FROM_HERE,
      base::Bind(&CertLoader::InitializeTokenAndLoadCertificates,
                 initialize_token_factory_.GetWeakPtr()),
      tpm_request_delay_);
  tpm_request_delay_ = GetNextRequestDelayMs(tpm_request_delay_);
}

// For background see this discussion on dev-tech-crypto.lists.mozilla.org:
// http://web.archiveorange.com/archive/v/6JJW7E40sypfZGtbkzxX
//
// NOTE: This function relies on the convention that the same PKCS#11 ID
// is shared between a certificate and its associated private and public
// keys.  I tried to implement this with PK11_GetLowLevelKeyIDForCert(),
// but that always returns NULL on Chrome OS for me.
std::string CertLoader::GetPkcs11IdForCert(
    const net::X509Certificate& cert) const {
  if (!IsHardwareBacked())
    return std::string();

  CERTCertificateStr* cert_handle = cert.os_cert_handle();
  SECKEYPrivateKey *priv_key =
      PK11_FindKeyByAnyCert(cert_handle, NULL /* wincx */);
  if (!priv_key)
    return std::string();

  // Get the CKA_ID attribute for a key.
  SECItem* sec_item = PK11_GetLowLevelKeyIDForPrivateKey(priv_key);
  std::string pkcs11_id;
  if (sec_item) {
    pkcs11_id = base::HexEncode(sec_item->data, sec_item->len);
    SECITEM_FreeItem(sec_item, PR_TRUE);
  }
  SECKEY_DestroyPrivateKey(priv_key);

  return pkcs11_id;
}

void CertLoader::OnTpmIsEnabled(DBusMethodCallStatus call_status,
                                bool tpm_is_enabled) {
  VLOG(1) << "OnTpmIsEnabled: " << tpm_is_enabled;

  if (call_status == DBUS_METHOD_CALL_SUCCESS && tpm_is_enabled)
    tpm_token_state_ = TPM_ENABLED;
  else
    tpm_token_state_ = TPM_DISABLED;

  InitializeTokenAndLoadCertificates();
}

void CertLoader::OnPkcs11IsTpmTokenReady(DBusMethodCallStatus call_status,
                                         bool is_tpm_token_ready) {
  VLOG(1) << "OnPkcs11IsTpmTokenReady: " << is_tpm_token_ready;

  if (call_status == DBUS_METHOD_CALL_FAILURE || !is_tpm_token_ready) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_state_ = TPM_TOKEN_READY;
  InitializeTokenAndLoadCertificates();
}

void CertLoader::OnPkcs11GetTpmTokenInfo(DBusMethodCallStatus call_status,
                                         const std::string& token_name,
                                         const std::string& user_pin) {
  VLOG(1) << "OnPkcs11GetTpmTokenInfo: " << token_name;

  if (call_status == DBUS_METHOD_CALL_FAILURE) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_name_ = token_name;
  // TODO(stevenjb): The network code expects a slot ID, not a label. See
  // crbug.com/201101. For now, use a hard coded, well known slot instead.
  const char kHardcodedTpmSlot[] = "0";
  tpm_token_slot_ = kHardcodedTpmSlot;
  tpm_user_pin_ = user_pin;
  tpm_token_state_ = TPM_TOKEN_INFO_RECEIVED;

  InitializeTokenAndLoadCertificates();
}

void CertLoader::InitializeNSSForTPMToken() {
  VLOG(1) << "InitializeNSSForTPMToken";

  if (base::chromeos::IsRunningOnChromeOS() &&
      !crypto::InitializeTPMToken(tpm_token_name_, tpm_user_pin_)) {
    RetryTokenInitializationLater();
    return;
  }

  tpm_token_state_ = TPM_TOKEN_NSS_INITIALIZED;
  InitializeTokenAndLoadCertificates();
}

void CertLoader::StartLoadCertificates() {
  VLOG(1) << "StartLoadCertificates";

  if (certificates_update_running_) {
    certificates_update_required_ = true;
    return;
  }

  net::CertificateList* cert_list = new net::CertificateList;
  certificates_update_running_ = true;
  certificates_update_required_ = false;
  base::WorkerPool::GetTaskRunner(true /* task_is_slow */)->
      PostTaskAndReply(
          FROM_HERE,
          base::Bind(LoadNSSCertificates, cert_list),
          base::Bind(&CertLoader::UpdateCertificates,
                     update_certificates_factory_.GetWeakPtr(),
                     base::Owned(cert_list)));
}

void CertLoader::UpdateCertificates(net::CertificateList* cert_list) {
  CHECK(thread_checker_.CalledOnValidThread());
  DCHECK(certificates_update_running_);
  VLOG(1) << "UpdateCertificates: " << cert_list->size();

  // Ignore any existing certificates.
  cert_list_.swap(*cert_list);

  NotifyCertificatesLoaded(!certificates_loaded_);
  certificates_loaded_ = true;

  certificates_update_running_ = false;
  if (certificates_update_required_)
    StartLoadCertificates();
}

void CertLoader::NotifyCertificatesLoaded(bool initial_load) {
  FOR_EACH_OBSERVER(Observer, observers_,
                    OnCertificatesLoaded(cert_list_, initial_load));
}

void CertLoader::OnCertTrustChanged(const net::X509Certificate* cert) {
}

void CertLoader::OnCertAdded(const net::X509Certificate* cert) {
  VLOG(1) << "OnCertAdded";
  StartLoadCertificates();
}

void CertLoader::OnCertRemoved(const net::X509Certificate* cert) {
  VLOG(1) << "OnCertRemoved";
  StartLoadCertificates();
}

void CertLoader::LoggedInStateChanged(LoginState::LoggedInState state) {
  VLOG(1) << "LoggedInStateChanged: " << state;
  RequestCertificates();
}

}  // namespace chromeos
<|endoftext|>


<commit_before>#ifndef TEST_LINEAR_H
#define TEST_LINEAR_H

#include "nnlib/nn/linear.hpp"
#include "test_module.hpp"
using namespace nnlib;

void TestLinear()
{
	// Linear layer with arbitrary parameters
	Linear<> module(2, 3);
	module.weights().copy({ -3, -2, 2, 3, 4, 5 });
	module.bias().copy({ -5, 7, 8862.37 });

	// Arbitrary input (batch)
	Tensor<> inp = Tensor<>({ -5, 10, 15, -20 }).resize(2, 2);

	// Arbitrary output gradient (batch)
	Tensor<> grd = Tensor<>({ 1, 2, 3, -4, -3, 2 }).resize(2, 3);

	// Output (fixed given input, weights, and bias)
	Tensor<> out = Tensor<>({ 40, 57, 8902.37, -110, -103, 8792.37 }).resize(2, 3);

	// Input gradient (fixed given input, weights, bias, and output gradient)
	Tensor<> ing = Tensor<>({ -1, 26, 22, -14 }).resize(2, 2);

	// Parameter gradient (fixed given input and output gradient)
	Tensor<> prg = Tensor<>({ -65, -55, 15, 90, 80, -10, -3, -1, 5 });

	// Test forward and backward using the parameters and targets above

	module.forward(inp);
	module.backward(inp, grd);

	NNAssertLessThan(module.output().copy().addM(out, -1).square().sum(), 1e-9, "Linear::forward failed; wrong output!");
	NNAssertLessThan(module.inGrad().copy().addM(ing, -1).square().sum(), 1e-9, "Linear::backward failed; wrong input gradient!");
	NNAssertLessThan(module.grad().addV(prg, -1).square().sum(), 1e-9, "Linear::backward failed; wrong parameter gradient!");

	module.forward(inp.select(0, 0));
	module.backward(inp.select(0, 0), grd.select(0, 0));

	NNAssertLessThan(module.output().copy().add(out.select(0, 0), -1).square().sum(), 1e-9, "Linear::forward failed for a vector; wrong output!");
	NNAssertLessThan(module.inGrad().copy().add(ing.select(0, 0), -1).square().sum(), 1e-9, "Linear::backward failed for a vector; wrong input gradient!");

	bool ok = true;
	try
	{
		module.forward(Tensor<>(1, 1, 1));
		ok = false;
	}
	catch(const Error &e) {}
	NNAssert(ok, "Linear::forward accepted an invalid input shape!");

	ok = true;
	try
	{
		module.backward(Tensor<>(1, 1, 1), Tensor<>(1, 1, 1));
		ok = false;
	}
	catch(const Error &e) {}
	NNAssert(ok, "Linear::backward accepted invalid input and outGrad shapes!");

	TestModule("Linear", module, inp);
}

#endif
<commit_msg>Added a unit test to linear for no-bias.<commit_after>#ifndef TEST_LINEAR_H
#define TEST_LINEAR_H

#include "nnlib/nn/linear.hpp"
#include "test_module.hpp"
using namespace nnlib;

void TestLinear()
{
	// Linear layer with arbitrary parameters
	Linear<> module(2, 3);
	module.weights().copy({ -3, -2, 2, 3, 4, 5 });
	module.bias().copy({ -5, 7, 8862.37 });

	// Arbitrary input (batch)
	Tensor<> inp = Tensor<>({ -5, 10, 15, -20 }).resize(2, 2);

	// Arbitrary output gradient (batch)
	Tensor<> grd = Tensor<>({ 1, 2, 3, -4, -3, 2 }).resize(2, 3);

	// Output (fixed given input, weights, and bias)
	Tensor<> out = Tensor<>({ 40, 57, 8902.37, -110, -103, 8792.37 }).resize(2, 3);

	// Input gradient (fixed given input, weights, bias, and output gradient)
	Tensor<> ing = Tensor<>({ -1, 26, 22, -14 }).resize(2, 2);

	// Parameter gradient (fixed given input and output gradient)
	Tensor<> prg = Tensor<>({ -65, -55, 15, 90, 80, -10, -3, -1, 5 });

	// Test forward and backward using the parameters and targets above

	module.forward(inp);
	module.backward(inp, grd);

	NNAssertLessThan(module.output().copy().addM(out, -1).square().sum(), 1e-9, "Linear::forward failed; wrong output!");
	NNAssertLessThan(module.inGrad().copy().addM(ing, -1).square().sum(), 1e-9, "Linear::backward failed; wrong input gradient!");
	NNAssertLessThan(module.grad().addV(prg, -1).square().sum(), 1e-9, "Linear::backward failed; wrong parameter gradient!");

	module.forward(inp.select(0, 0));
	module.backward(inp.select(0, 0), grd.select(0, 0));

	NNAssertLessThan(module.output().copy().add(out.select(0, 0), -1).square().sum(), 1e-9, "Linear::forward failed for a vector; wrong output!");
	NNAssertLessThan(module.inGrad().copy().add(ing.select(0, 0), -1).square().sum(), 1e-9, "Linear::backward failed for a vector; wrong input gradient!");

	module.useBias(false);
	module.forward(inp.select(0, 0));
	module.backward(inp.select(0, 0), grd.select(0, 0));

	NNAssertLessThan(module.output().copy().add(module.bias()).add(out.select(0, 0), -1).square().sum(), 1e-9, "Linear::forward failed without bias; wrong output!");
	NNAssertLessThan(module.inGrad().copy().add(ing.select(0, 0), -1).square().sum(), 1e-9, "Linear::backward failed without bias; wrong input gradient!");

	bool ok = true;
	try
	{
		module.forward(Tensor<>(1, 1, 1));
		ok = false;
	}
	catch(const Error &e) {}
	NNAssert(ok, "Linear::forward accepted an invalid input shape!");

	ok = true;
	try
	{
		module.backward(Tensor<>(1, 1, 1), Tensor<>(1, 1, 1));
		ok = false;
	}
	catch(const Error &e) {}
	NNAssert(ok, "Linear::backward accepted invalid input and outGrad shapes!");

	TestModule("Linear", module, inp);
}

#endif
<|endoftext|>


<commit_before>#include "gtest/gtest.h"

#include <fstream>

#include "../test_common.h"

#ifndef BIN_PATH
#error require BIN_PATH
#endif

using namespace ydsh;

#define EACH_DUMP_OP(OP) \
    OP(untyped, "--dump-untyped-ast") \
    OP(typed,  "--dump-ast")

enum class DumpOp {
#define GEN_ENUM(E, S) E,
    EACH_DUMP_OP(GEN_ENUM)
#undef GEN_ENUM
};

static const char *toString(DumpOp op) {
    const char *table[] = {
#define GEN_STR(E, S) #E,
            EACH_DUMP_OP(GEN_STR)
#undef GEN_STR
    };
    return table[static_cast<unsigned int>(op)];
}

static const char *toOption(DumpOp op) {
    const char *table[] = {
#define GEN_STR(E, S) S,
        EACH_DUMP_OP(GEN_STR)
#undef GEN_STR
    };
    return table[static_cast<unsigned int>(op)];
}

struct NodeDumpParam {
    DumpOp op;
    const char *cmd;
    unsigned int local;
    unsigned int global;
    const char *dump;
};

class NodeDumpTest : public ::testing::TestWithParam<NodeDumpParam>, public TempFileFactory {
protected:
    static constexpr unsigned int GVAR_NUM = 50;

    NodeDumpParam param;

public:
    NodeDumpTest() {
        this->param = this->GetParam();
    }

    ~NodeDumpTest() override = default;

protected:
    void readContent(std::string &value) {
        std::ifstream input(this->getTempFileName());
        ASSERT_FALSE(!input);

        for(std::string line; std::getline(input, line); ) {
            value += line;
            value += '\n';
        }
    }

    void test() {
        std::string dump = toOption(this->param.op);
        dump += "=";
        dump += this->getTempFileName();

        auto ret = ProcBuilder {
                BIN_PATH,
                dump.c_str(),
                "--compile-only",
                "-c",
                this->param.cmd
        }.exec();

        ASSERT_EQ(WaitStatus::EXITED, ret.kind);
        ASSERT_EQ(0, ret.value);

        auto expect = this->formatOutput();
        std::string actual;
        ASSERT_NO_FATAL_FAILURE(this->readContent(actual));

        ASSERT_EQ(expect, actual);
    }

private:
    static const char *skipFirstSpace(const char *str) {
        for(; *str != '\0' && (*str == ' ' || *str == '\t' || *str == '\n'); str++);
        return str;
    }

    std::string formatOutput() const {
        auto value = format(R"EOF(### dump %s AST ###
sourceName: "(string)"
)EOF", toString(this->param.op));

        value += skipFirstSpace(this->param.dump);

        // skip last spaces
        for(; !value.empty(); value.pop_back()) {
            char ch = value.back();
            if(ch != ' ' && ch != '\t' && ch != '\n') {
                break;
            }
        }

        value += format(R"EOF(
maxVarNum: "%d"
maxGVarNum: "%d"

)EOF", this->param.local, GVAR_NUM + this->param.global);

        return value;
    }
};

// test parameter definition
static constexpr NodeDumpParam paramTable[] = {
        {DumpOp::untyped, "1", 0, 0, R"(
nodes:
  - nodeKind: Number
    token:
      pos: 0
      size: 1
    type:
    kind: "Int32"
    intValue: "1"
)"},

        {DumpOp::typed, R"("hello")", 0, 0, R"(
nodes:
  - nodeKind: TypeOp
    token:
      pos: 0
      size: 7
    type: Void
    exprNode:
      nodeKind: StringExpr
      token:
        pos: 0
        size: 7
      type: String
      nodes:
        - nodeKind: String
          token:
            pos: 1
            size: 5
          type: String
          kind: "STRING"
          value: "hello"
    targetTypeToken:
    opKind: "TO_VOID"

)"},

        {DumpOp::untyped, R"(try { var a = 'false'; } catch $e {} finally {1l; })", 1, 0, R"(
nodes:
  - nodeKind: Try
    token:
      pos: 0
      size: 51
    type:
    exprNode:
      nodeKind: Block
      token:
        pos: 4
        size: 20
      type:
      nodes:
        - nodeKind: VarDecl
          token:
            pos: 6
            size: 15
          type:
          varName: "a"
          global: "0"
          varIndex: "0"
          exprNode:
            nodeKind: String
            token:
              pos: 14
              size: 7
            type:
            kind: "STRING"
            value: "false"
          kind: "VAR"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
    catchNodes:
      - nodeKind: Catch
        token:
          pos: 25
          size: 11
        type:
        exceptionName: "e"
        typeNode:
          nodeKind: Type
          token:
            pos: 0
            size: 0
          type:
          typeKind: "Base"
          typeName: "Any"
        blockNode:
          nodeKind: Block
          token:
            pos: 34
            size: 2
          type:
          nodes:
          baseIndex: "0"
          varSize: "0"
          maxVarSize: "0"
        varIndex: "0"
    finallyNode:
      nodeKind: Block
      token:
        pos: 45
        size: 6
      type:
      nodes:
        - nodeKind: Number
          token:
            pos: 46
            size: 2
          type:
          kind: "Int64"
          longValue: "1"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
)"},

        {DumpOp::typed, R"({;})", 0, 0, R"(
nodes:
  - nodeKind: Block
    token:
      pos: 0
      size: 3
    type: Void
    nodes:
      - nodeKind: Empty
        token:
          pos: 1
          size: 1
        type: Void
    baseIndex: "0"
    varSize: "0"
    maxVarSize: "0"
)"},

        {DumpOp::untyped, R"('hey'.size())", 0, 0, R"(
nodes:
  - nodeKind: Apply
    token:
      pos: 0
      size: 12
    type:
    exprNode:
      nodeKind: Access
      token:
        pos: 0
        size: 5
      type:
      recvNode:
        nodeKind: String
        token:
          pos: 0
          size: 5
        type:
        kind: "STRING"
        value: "hey"
      nameNode:
        nodeKind: Var
        token:
          pos: 6
          size: 4
        type:
        varName: "size"
        index: "0"
        attribute: ""
      index: "0"
      attribute: ""
      additionalOp: "NOP"
    argNodes:
    methodIndex: "0"
    kind: "UNRESOLVED"
)"},

        {DumpOp::untyped, R"(34+1)", 0, 0, R"(
nodes:
  - nodeKind: BinaryOp
    token:
      pos: 0
      size: 4
    type:
    leftNode:
      nodeKind: Number
      token:
        pos: 0
        size: 2
      type:
      kind: "Int32"
      intValue: "34"
    rightNode:
      nodeKind: Number
      token:
        pos: 3
        size: 1
      type:
      kind: "Int32"
      intValue: "1"
    op: "+"
    optNode:
)"},

        {DumpOp::untyped, R"(function f() : typeof($/d/) {return new Regex("$true"); })", 0, 1, R"(
nodes:
  - nodeKind: Function
    token:
      pos: 0
      size: 57
    type:
    funcName: "f"
    paramNodes:
    paramTypeNodes:
    returnTypeNode:
      nodeKind: Type
      token:
        pos: 15
        size: 12
      type:
      typeKind: "Return"
      typeNodes:
        - nodeKind: Type
          token:
            pos: 15
            size: 12
          type:
          typeKind: "TypeOf"
          exprNode:
            nodeKind: Regex
            token:
              pos: 22
              size: 4
            type:
            reStr: "d"
    blockNode:
      nodeKind: Block
      token:
        pos: 28
        size: 29
      type:
      nodes:
        - nodeKind: Jump
          token:
            pos: 29
            size: 25
          type:
          opKind: "RETURN"
          exprNode:
            nodeKind: New
            token:
              pos: 36
              size: 18
            type:
            targetTypeNode:
              nodeKind: Type
              token:
                pos: 40
                size: 5
              type:
              typeKind: "Base"
              typeName: "Regex"
            argNodes:
              - nodeKind: StringExpr
                token:
                  pos: 46
                  size: 7
                type:
                nodes:
                  - nodeKind: Embed
                    token:
                      pos: 47
                      size: 5
                    type:
                    kind: "STR_EXPR"
                    exprNode:
                      nodeKind: Var
                      token:
                        pos: 47
                        size: 5
                      type:
                      varName: "true"
                      index: "0"
                      attribute: ""
                    methodIndex: "0"
          leavingBlock: "0"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
    maxVarNum: "0"
    varIndex: "0"
    funcType:
)"},

        {DumpOp::untyped, R"(assert (!ls > 34 | 34 with < ${34.1} &).poll())", 2, 0, R"(
nodes:
  - nodeKind: Assert
    token:
      pos: 0
      size: 1
    type:
    condNode:
      nodeKind: Apply
      token:
        pos: 7
        size: 39
      type:
      exprNode:
        nodeKind: Access
        token:
          pos: 7
          size: 32
        type:
        recvNode:
          nodeKind: Fork
          token:
            pos: 7
            size: 32
          type:
          exprNode:
            nodeKind: Pipeline
            token:
              pos: 8
              size: 28
            type:
            nodes:
              - nodeKind: UnaryOp
                token:
                  pos: 8
                  size: 8
                type:
                op: "!"
                exprNode:
                  nodeKind: Cmd
                  token:
                    pos: 9
                    size: 7
                  type:
                  nameNode:
                    nodeKind: String
                    token:
                      pos: 9
                      size: 2
                    type:
                    kind: "STRING"
                    value: "ls"
                  argNodes:
                    - nodeKind: Redir
                      token:
                        pos: 14
                        size: 2
                      type:
                      op: "1>"
                      targetNode:
                        nodeKind: CmdArg
                        token:
                          pos: 14
                          size: 2
                        type:
                        segmentNodes:
                          - nodeKind: String
                            token:
                              pos: 14
                              size: 2
                            type:
                            kind: "STRING"
                            value: "34"
                  redirCount: "1"
                  inPipe: "0"
                methodCallNode:
              - nodeKind: With
                token:
                  pos: 19
                  size: 17
                type:
                exprNode:
                  nodeKind: Number
                  token:
                    pos: 19
                    size: 2
                  type:
                  kind: "Int32"
                  intValue: "34"
                redirNodes:
                  - nodeKind: Redir
                    token:
                      pos: 29
                      size: 7
                    type:
                    op: "<"
                    targetNode:
                      nodeKind: CmdArg
                      token:
                        pos: 29
                        size: 7
                      type:
                      segmentNodes:
                        - nodeKind: Embed
                          token:
                            pos: 29
                            size: 7
                          type:
                          kind: "CMD_ARG"
                          exprNode:
                            nodeKind: Number
                            token:
                              pos: 31
                              size: 4
                            type:
                            kind: "Float"
                            floatValue: "34.100000"
                          methodIndex: "0"
                baseIndex: "0"
            baseIndex: "0"
          opKind: "ForkKind::JOB"
        nameNode:
          nodeKind: Var
          token:
            pos: 40
            size: 4
          type:
          varName: "poll"
          index: "0"
          attribute: ""
        index: "0"
        attribute: ""
        additionalOp: "NOP"
      argNodes:
      methodIndex: "0"
      kind: "UNRESOLVED"
    messageNode:
      nodeKind: String
      token:
        pos: 0
        size: 0
      type:
      kind: "STRING"
      value: "`(!ls > 34 | 34 with < ${34.1} &).poll()'"
)"},

        {DumpOp::typed, R"(case %'int' { %'int' => [34:34]; else => (34,)})", 0, 0, R"(
nodes:
  - nodeKind: Case
    token:
      pos: 0
      size: 47
    type: Void
    exprNode:
      nodeKind: Number
      token:
        pos: 5
        size: 6
      type: Signal
      kind: "Signal"
      intValue: "2"
    armNodes:
      - nodeKind: Arm
        token:
          pos: 14
          size: 17
        type: Void
        this->patternNodes:
          - nodeKind: Number
            token:
              pos: 14
              size: 6
            type: Signal
            kind: "Signal"
            intValue: "2"
        this->actionNode:
          nodeKind: TypeOp
          token:
            pos: 24
            size: 7
          type: Void
          exprNode:
            nodeKind: Map
            token:
              pos: 24
              size: 7
            type: [Int32 : Int32]
            keyNodes:
              - nodeKind: Number
                token:
                  pos: 25
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
            valueNodes:
              - nodeKind: Number
                token:
                  pos: 28
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
          targetTypeToken:
          opKind: "TO_VOID"
      - nodeKind: Arm
        token:
          pos: 33
          size: 13
        type: Void
        this->patternNodes:
        this->actionNode:
          nodeKind: TypeOp
          token:
            pos: 41
            size: 5
          type: Void
          exprNode:
            nodeKind: Tuple
            token:
              pos: 41
              size: 5
            type: (Int32,)
            nodes:
              - nodeKind: Number
                token:
                  pos: 42
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
          targetTypeToken:
          opKind: "TO_VOID"
    caseKind: "MAP"

)"},

        {DumpOp::untyped, R"(alias i = (Int) -> Int)", 0, 0, R"(
nodes:
  - nodeKind: TypeAlias
    token:
      pos: 0
      size: 22
    type:
    alias: "i"
    targetTypeNode:
      nodeKind: Type
      token:
        pos: 10
        size: 12
      type:
      typeKind: "Func"
      returnTypeNode:
        nodeKind: Type
        token:
          pos: 19
          size: 3
        type:
        typeKind: "Base"
        typeName: "Int"
      paramTypeNodes:
        - nodeKind: Type
          token:
            pos: 11
            size: 3
          type:
          typeKind: "Base"
          typeName: "Int"
)"},
};

TEST_P(NodeDumpTest, base) {
    ASSERT_NO_FATAL_FAILURE(this->test());
}

INSTANTIATE_TEST_SUITE_P(NodeDumpTest, NodeDumpTest, ::testing::ValuesIn(paramTable));

int main(int argc, char **argv) {
    ::testing::InitGoogleTest(&argc, argv);
    return RUN_ALL_TESTS();
}<commit_msg>update test case<commit_after>#include "gtest/gtest.h"

#include <fstream>

#include "../test_common.h"

#ifndef BIN_PATH
#error require BIN_PATH
#endif

using namespace ydsh;

#define EACH_DUMP_OP(OP) \
    OP(untyped, "--dump-untyped-ast") \
    OP(typed,  "--dump-ast")

enum class DumpOp {
#define GEN_ENUM(E, S) E,
    EACH_DUMP_OP(GEN_ENUM)
#undef GEN_ENUM
};

static const char *toString(DumpOp op) {
    const char *table[] = {
#define GEN_STR(E, S) #E,
            EACH_DUMP_OP(GEN_STR)
#undef GEN_STR
    };
    return table[static_cast<unsigned int>(op)];
}

static const char *toOption(DumpOp op) {
    const char *table[] = {
#define GEN_STR(E, S) S,
        EACH_DUMP_OP(GEN_STR)
#undef GEN_STR
    };
    return table[static_cast<unsigned int>(op)];
}

struct NodeDumpParam {
    DumpOp op;
    const char *cmd;
    unsigned int local;
    unsigned int global;
    const char *dump;
};

class NodeDumpTest : public ::testing::TestWithParam<NodeDumpParam>, public TempFileFactory {
protected:
    static constexpr unsigned int GVAR_NUM = 50;

    NodeDumpParam param;

public:
    NodeDumpTest() {
        this->param = this->GetParam();
    }

    ~NodeDumpTest() override = default;

protected:
    void readContent(std::string &value) {
        std::ifstream input(this->getTempFileName());
        ASSERT_FALSE(!input);

        for(std::string line; std::getline(input, line); ) {
            value += line;
            value += '\n';
        }
    }

    void test() {
        std::string dump = toOption(this->param.op);
        dump += "=";
        dump += this->getTempFileName();

        auto ret = ProcBuilder {
                BIN_PATH,
                dump.c_str(),
                "--compile-only",
                "-c",
                this->param.cmd
        }.exec();

        ASSERT_EQ(WaitStatus::EXITED, ret.kind);
        ASSERT_EQ(0, ret.value);

        auto expect = this->formatOutput();
        std::string actual;
        ASSERT_NO_FATAL_FAILURE(this->readContent(actual));

        ASSERT_EQ(expect, actual);
    }

private:
    static const char *skipFirstSpace(const char *str) {
        for(; *str != '\0' && (*str == ' ' || *str == '\t' || *str == '\n'); str++);
        return str;
    }

    std::string formatOutput() const {
        auto value = format(R"EOF(### dump %s AST ###
sourceName: "(string)"
)EOF", toString(this->param.op));

        value += skipFirstSpace(this->param.dump);

        // skip last spaces
        for(; !value.empty(); value.pop_back()) {
            char ch = value.back();
            if(ch != ' ' && ch != '\t' && ch != '\n') {
                break;
            }
        }

        value += format(R"EOF(
maxVarNum: "%d"
maxGVarNum: "%d"

)EOF", this->param.local, GVAR_NUM + this->param.global);

        return value;
    }
};

// test parameter definition
static constexpr NodeDumpParam paramTable[] = {
        {DumpOp::untyped, "1", 0, 0, R"(
nodes:
  - nodeKind: Number
    token:
      pos: 0
      size: 1
    type:
    kind: "Int32"
    intValue: "1"
)"},

        {DumpOp::typed, R"("hello")", 0, 0, R"(
nodes:
  - nodeKind: TypeOp
    token:
      pos: 0
      size: 7
    type: Void
    exprNode:
      nodeKind: StringExpr
      token:
        pos: 0
        size: 7
      type: String
      nodes:
        - nodeKind: String
          token:
            pos: 1
            size: 5
          type: String
          kind: "STRING"
          value: "hello"
    targetTypeToken:
    opKind: "TO_VOID"

)"},

        {DumpOp::untyped, R"(try { var a = 'false'; } catch $e {} finally {1l; })", 1, 0, R"(
nodes:
  - nodeKind: Try
    token:
      pos: 0
      size: 51
    type:
    exprNode:
      nodeKind: Block
      token:
        pos: 4
        size: 20
      type:
      nodes:
        - nodeKind: VarDecl
          token:
            pos: 6
            size: 15
          type:
          varName: "a"
          global: "0"
          varIndex: "0"
          exprNode:
            nodeKind: String
            token:
              pos: 14
              size: 7
            type:
            kind: "STRING"
            value: "false"
          kind: "VAR"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
    catchNodes:
      - nodeKind: Catch
        token:
          pos: 25
          size: 11
        type:
        exceptionName: "e"
        typeNode:
          nodeKind: Type
          token:
            pos: 0
            size: 0
          type:
          typeKind: "Base"
          typeName: "Any"
        blockNode:
          nodeKind: Block
          token:
            pos: 34
            size: 2
          type:
          nodes:
          baseIndex: "0"
          varSize: "0"
          maxVarSize: "0"
        varIndex: "0"
    finallyNode:
      nodeKind: Block
      token:
        pos: 45
        size: 6
      type:
      nodes:
        - nodeKind: Number
          token:
            pos: 46
            size: 2
          type:
          kind: "Int64"
          longValue: "1"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
)"},

        {DumpOp::typed, R"({;})", 0, 0, R"(
nodes:
  - nodeKind: Block
    token:
      pos: 0
      size: 3
    type: Void
    nodes:
      - nodeKind: Empty
        token:
          pos: 1
          size: 1
        type: Void
    baseIndex: "0"
    varSize: "0"
    maxVarSize: "0"
)"},

        {DumpOp::untyped, R"('hey'.size())", 0, 0, R"(
nodes:
  - nodeKind: Apply
    token:
      pos: 0
      size: 12
    type:
    exprNode:
      nodeKind: Access
      token:
        pos: 0
        size: 5
      type:
      recvNode:
        nodeKind: String
        token:
          pos: 0
          size: 5
        type:
        kind: "STRING"
        value: "hey"
      nameNode:
        nodeKind: Var
        token:
          pos: 6
          size: 4
        type:
        varName: "size"
        index: "0"
        attribute: ""
      index: "0"
      attribute: ""
      additionalOp: "NOP"
    argNodes:
    methodIndex: "0"
    kind: "UNRESOLVED"
)"},

        {DumpOp::untyped, R"(34+1)", 0, 0, R"(
nodes:
  - nodeKind: BinaryOp
    token:
      pos: 0
      size: 4
    type:
    leftNode:
      nodeKind: Number
      token:
        pos: 0
        size: 2
      type:
      kind: "Int32"
      intValue: "34"
    rightNode:
      nodeKind: Number
      token:
        pos: 3
        size: 1
      type:
      kind: "Int32"
      intValue: "1"
    op: "+"
    optNode:
)"},

        {DumpOp::untyped, R"(function f() : typeof($/d/) {return new Regex("$true"); })", 0, 1, R"(
nodes:
  - nodeKind: Function
    token:
      pos: 0
      size: 57
    type:
    funcName: "f"
    paramNodes:
    paramTypeNodes:
    returnTypeNode:
      nodeKind: Type
      token:
        pos: 15
        size: 12
      type:
      typeKind: "Return"
      typeNodes:
        - nodeKind: Type
          token:
            pos: 15
            size: 12
          type:
          typeKind: "TypeOf"
          exprNode:
            nodeKind: Regex
            token:
              pos: 22
              size: 4
            type:
            reStr: "d"
    blockNode:
      nodeKind: Block
      token:
        pos: 28
        size: 29
      type:
      nodes:
        - nodeKind: Jump
          token:
            pos: 29
            size: 25
          type:
          opKind: "RETURN"
          exprNode:
            nodeKind: New
            token:
              pos: 36
              size: 18
            type:
            targetTypeNode:
              nodeKind: Type
              token:
                pos: 40
                size: 5
              type:
              typeKind: "Base"
              typeName: "Regex"
            argNodes:
              - nodeKind: StringExpr
                token:
                  pos: 46
                  size: 7
                type:
                nodes:
                  - nodeKind: Embed
                    token:
                      pos: 47
                      size: 5
                    type:
                    kind: "STR_EXPR"
                    exprNode:
                      nodeKind: Var
                      token:
                        pos: 47
                        size: 5
                      type:
                      varName: "true"
                      index: "0"
                      attribute: ""
                    methodIndex: "0"
          leavingBlock: "0"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
    maxVarNum: "0"
    varIndex: "0"
    funcType:
)"},

        {DumpOp::untyped, R"(assert (!ls > 34 | 34 with < ${34.1} &).poll())", 2, 0, R"(
nodes:
  - nodeKind: Assert
    token:
      pos: 0
      size: 1
    type:
    condNode:
      nodeKind: Apply
      token:
        pos: 7
        size: 39
      type:
      exprNode:
        nodeKind: Access
        token:
          pos: 7
          size: 32
        type:
        recvNode:
          nodeKind: Fork
          token:
            pos: 7
            size: 32
          type:
          exprNode:
            nodeKind: Pipeline
            token:
              pos: 8
              size: 28
            type:
            nodes:
              - nodeKind: UnaryOp
                token:
                  pos: 8
                  size: 8
                type:
                op: "!"
                exprNode:
                  nodeKind: Cmd
                  token:
                    pos: 9
                    size: 7
                  type:
                  nameNode:
                    nodeKind: String
                    token:
                      pos: 9
                      size: 2
                    type:
                    kind: "STRING"
                    value: "ls"
                  argNodes:
                    - nodeKind: Redir
                      token:
                        pos: 14
                        size: 2
                      type:
                      op: "1>"
                      targetNode:
                        nodeKind: CmdArg
                        token:
                          pos: 14
                          size: 2
                        type:
                        segmentNodes:
                          - nodeKind: String
                            token:
                              pos: 14
                              size: 2
                            type:
                            kind: "STRING"
                            value: "34"
                  redirCount: "1"
                  inPipe: "0"
                methodCallNode:
              - nodeKind: With
                token:
                  pos: 19
                  size: 17
                type:
                exprNode:
                  nodeKind: Number
                  token:
                    pos: 19
                    size: 2
                  type:
                  kind: "Int32"
                  intValue: "34"
                redirNodes:
                  - nodeKind: Redir
                    token:
                      pos: 29
                      size: 7
                    type:
                    op: "<"
                    targetNode:
                      nodeKind: CmdArg
                      token:
                        pos: 29
                        size: 7
                      type:
                      segmentNodes:
                        - nodeKind: Embed
                          token:
                            pos: 29
                            size: 7
                          type:
                          kind: "CMD_ARG"
                          exprNode:
                            nodeKind: Number
                            token:
                              pos: 31
                              size: 4
                            type:
                            kind: "Float"
                            floatValue: "34.100000"
                          methodIndex: "0"
                baseIndex: "0"
            baseIndex: "0"
          opKind: "ForkKind::JOB"
        nameNode:
          nodeKind: Var
          token:
            pos: 40
            size: 4
          type:
          varName: "poll"
          index: "0"
          attribute: ""
        index: "0"
        attribute: ""
        additionalOp: "NOP"
      argNodes:
      methodIndex: "0"
      kind: "UNRESOLVED"
    messageNode:
      nodeKind: String
      token:
        pos: 0
        size: 0
      type:
      kind: "STRING"
      value: "`(!ls > 34 | 34 with < ${34.1} &).poll()'"
)"},

        {DumpOp::typed, R"(case %'int' { %'int' => [34:34]; else => (34,)})", 0, 0, R"(
nodes:
  - nodeKind: Case
    token:
      pos: 0
      size: 47
    type: Void
    exprNode:
      nodeKind: Number
      token:
        pos: 5
        size: 6
      type: Signal
      kind: "Signal"
      intValue: "2"
    armNodes:
      - nodeKind: Arm
        token:
          pos: 14
          size: 17
        type: Void
        this->patternNodes:
          - nodeKind: Number
            token:
              pos: 14
              size: 6
            type: Signal
            kind: "Signal"
            intValue: "2"
        this->actionNode:
          nodeKind: TypeOp
          token:
            pos: 24
            size: 7
          type: Void
          exprNode:
            nodeKind: Map
            token:
              pos: 24
              size: 7
            type: [Int32 : Int32]
            keyNodes:
              - nodeKind: Number
                token:
                  pos: 25
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
            valueNodes:
              - nodeKind: Number
                token:
                  pos: 28
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
          targetTypeToken:
          opKind: "TO_VOID"
      - nodeKind: Arm
        token:
          pos: 33
          size: 13
        type: Void
        this->patternNodes:
        this->actionNode:
          nodeKind: TypeOp
          token:
            pos: 41
            size: 5
          type: Void
          exprNode:
            nodeKind: Tuple
            token:
              pos: 41
              size: 5
            type: (Int32,)
            nodes:
              - nodeKind: Number
                token:
                  pos: 42
                  size: 2
                type: Int32
                kind: "Int32"
                intValue: "34"
          targetTypeToken:
          opKind: "TO_VOID"
    caseKind: "MAP"

)"},

        {DumpOp::untyped, R"(alias i = (Int) -> Int)", 0, 0, R"(
nodes:
  - nodeKind: TypeAlias
    token:
      pos: 0
      size: 22
    type:
    alias: "i"
    targetTypeNode:
      nodeKind: Type
      token:
        pos: 10
        size: 12
      type:
      typeKind: "Func"
      returnTypeNode:
        nodeKind: Type
        token:
          pos: 19
          size: 3
        type:
        typeKind: "Base"
        typeName: "Int"
      paramTypeNodes:
        - nodeKind: Type
          token:
            pos: 11
            size: 3
          type:
          typeKind: "Base"
          typeName: "Int"
)"},

        {DumpOp::typed, R"(while($false){})", 0, 0, R"(
nodes:
  - nodeKind: Loop
    token:
      pos: 0
      size: 15
    type: Void
    initNode:
      nodeKind: Empty
      token:
        pos: 0
        size: 0
      type: Void
    condNode:
      nodeKind: Var
      token:
        pos: 5
        size: 8
      type: Boolean
      varName: "false"
      index: "42"
      attribute: "READ_ONLY | GLOBAL | BUILTIN"
    iterNode:
      nodeKind: Empty
      token:
        pos: 0
        size: 0
      type: Void
    blockNode:
      nodeKind: Block
      token:
        pos: 13
        size: 2
      type: Nothing
      nodes:
        - nodeKind: Jump
          token:
            pos: 0
            size: 0
          type: Nothing
          opKind: "CONTINUE"
          exprNode:
            nodeKind: Empty
            token:
              pos: 0
              size: 0
            type: Void
          leavingBlock: "0"
      baseIndex: "0"
      varSize: "0"
      maxVarSize: "0"
    asDoWhile: "0"
)"},
};

TEST_P(NodeDumpTest, base) {
    ASSERT_NO_FATAL_FAILURE(this->test());
}

INSTANTIATE_TEST_SUITE_P(NodeDumpTest, NodeDumpTest, ::testing::ValuesIn(paramTable));

int main(int argc, char **argv) {
    ::testing::InitGoogleTest(&argc, argv);
    return RUN_ALL_TESTS();
}<|endoftext|>


<commit_before>/*******************************************************************************
 * Copyright (c) 2012, Dougal J. Sutherland (dsutherl@cs.cmu.edu).             *
 * All rights reserved.                                                        *
 *                                                                             *
 * Redistribution and use in source and binary forms, with or without          *
 * modification, are permitted provided that the following conditions are met: *
 *                                                                             *
 *     * Redistributions of source code must retain the above copyright        *
 *       notice, this list of conditions and the following disclaimer.         *
 *                                                                             *
 *     * Redistributions in binary form must reproduce the above copyright     *
 *       notice, this list of conditions and the following disclaimer in the   *
 *       documentation and/or other materials provided with the distribution.  *
 *                                                                             *
 *     * Neither the name of Carnegie Mellon University nor the                *
 *       names of the contributors may be used to endorse or promote products  *
 *       derived from this software without specific prior written permission. *
 *                                                                             *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" *
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE   *
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE  *
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE   *
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR         *
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF        *
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS    *
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN     *
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)     *
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE  *
 * POSSIBILITY OF SUCH DAMAGE.                                                 *
 ******************************************************************************/
#include "np-divs/np_divs.hpp"
#include "np-divs/matrix_io.hpp"
#include "np-divs/div-funcs/div_func.hpp"
#include "np-divs/div-funcs/from_str.hpp"

#include <iostream>
#include <fstream>
#include <string>

#include <boost/bind.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/program_options.hpp>
#include <boost/ptr_container/ptr_vector.hpp>
#include <boost/utility.hpp>

#include <flann/util/matrix.h>

using namespace npdivs;
using namespace std;
using namespace boost;
namespace po = boost::program_options;

struct ProgOpts : noncopyable {
    string x_bags_file;
    string y_bags_file;
    string results_file;

    ptr_vector<DivFunc> div_funcs;

    size_t k;
    size_t num_threads;

    flann::IndexParams index_params;
    flann::SearchParams search_params;

    size_t show_progress;

    void parse_div_funcs(const vector<string> &names) {
        for (size_t i = 0; i < names.size(); i++) {
            div_funcs.push_back(div_func_from_str(names[i]));
        }
    }

    void parse_index(const string name) {
        index_params = index_params_from_str(name);
    }
};

bool parse_args(int argc, char ** argv, ProgOpts& opts);

int main(int argc, char ** argv) {
    typedef flann::Matrix<double> Matrix;

    try {
        ProgOpts opts;
        opts.search_params = flann::SearchParams(64);
        if (!parse_args(argc, argv, opts))
            return 1;

        size_t num_df = opts.div_funcs.size();
        if (num_df == 0) {
            cerr << "Error: at least one div func is required\n";
            return 1;
        }

        // load input bags
        // TODO - gracefully handle nonexisting files
        size_t num_x;
        Matrix* x_bags;
        if (opts.x_bags_file == "-") {
            x_bags = matrices_from_csv(cin, num_x);
        } else {
            ifstream ifs(opts.x_bags_file.c_str(), ifstream::in);
            x_bags = matrices_from_csv(ifs, num_x);
            if (opts.show_progress)
                cerr << "Read " << num_x << " x bags.\n";
        }

        size_t num_y;
        Matrix* y_bags;
        if (opts.y_bags_file.empty()) {
            y_bags = NULL;
            num_y = num_x;
        } else if (opts.y_bags_file == "-") {
            y_bags = matrices_from_csv(cin, num_y);
        } else {
            ifstream ifs(opts.y_bags_file.c_str(), ifstream::in);
            y_bags = matrices_from_csv(ifs, num_y);
            if (opts.show_progress)
                cerr << "Read " << num_x << " y bags.\n";
        }

        Matrix* results = alloc_matrix_array<double>(num_df, num_x, num_y);

        DivParams params(opts.k, opts.index_params, opts.search_params,
                opts.num_threads, opts.show_progress);

        boost::posix_time::ptime t_start = boost::posix_time::second_clock::local_time();

        np_divs(x_bags, num_x, y_bags, num_y, opts.div_funcs, results, params);

        boost::posix_time::ptime t_end = boost::posix_time::second_clock::local_time();
        if (opts.show_progress)
            cerr << "Computation took " << (t_end - t_start).total_seconds() << " seconds.\n";

        if (opts.results_file == "-") {
            matrix_array_to_csv(cout, results, num_df);
        } else {
            ofstream ofs(opts.results_file.c_str());
            matrix_array_to_csv(ofs, results, num_df);
        }

        free_matrix_array(results, num_df);

    } catch (std::exception &e) {
        cerr << "Error: " << e.what() << endl;
        exit(1);
    }

    return 0;
}

// TODO nicer handling of matrix inputs
// TODO optionally support HDF5 inputs
// TODO positional arguments for x_bags, y_bags
// TODO support setting {index,search}_params
bool parse_args(int argc, char ** argv, ProgOpts& opts) {
    po::options_description desc("Allowed options");
    desc.add_options()
        ("help,h", "Produce this help message.")
        ("x-bags,x",
            po::value<string>(&opts.x_bags_file)->default_value("-"),
            "CSV-style file containing matrices separated by blank lines; "
            "- means stdin.")
        ("y-bags,y",
            po::value<string>(&opts.y_bags_file),
            "CSV-style file containing matrices separated by blank lines. "
            "If passed, divergences are calculated between the x bags (rows "
            "in the output matrix) and the y bags (columns); if not passed, "
            "divergences are calculated from the x bags to themselves. Pass "
            "- to use stdin; if both x and y are from stdin, x is read first "
            "and the two groups must be separated by exactly two blank lines. ")
        ("results,r",
            po::value<string>(&opts.results_file)->default_value("-"),
            "Where to write CSV-style matrix of results, where m[i,j] = "
            "div(x_i, y_j). Use - for stdout.")
        ("div-func,f",
            po::value< vector<string> >()->composing()
               ->notifier(bind(&ProgOpts::parse_div_funcs, ref(opts), _1)),
            "Divergence functions to use; can be specified more than once. If "
            "none passed, uses L2. Format is name:arg1:arg2:..., where argN "
            "refers to the Nth argument to the corresponding DivFunc's "
            "constructor. Some support a first argument specifying a "
            "parameter: renyi:.99 means the Renyi-.99 divergence. All support "
            "a last parameter, which determines the way that large values are "
            "normalized: l2:.95 or renyi:.99:.95 means certain calculated "
            "intermediate values above the 95th percentile are cut down; 1 "
            "means not to do this; default is .99. All extra arguments are "
            "optional.")
        ("num-threads",
            po::value<size_t>(&opts.num_threads)->default_value(0),
            "Number of threads to use for calculations. 0 means one per core.")
        ("neighbors,k",
            po::value<size_t>(&opts.k)->default_value(3),
            "The k for k-nearest-neighbor calculations.")
        ("index,i",
            po::value<string>()->default_value("kdtree")
                ->notifier(bind(&ProgOpts::parse_index, ref(opts), _1)),
            "The nearest-neighbor index to use. Options: linear, kdtree.")
        ("progress,p",
            po::value<size_t>(&opts.show_progress)->default_value(1000),
            "Show progress indications every X computations (default 1000; "
            "use 0 to not show any).")
    ;

    po::variables_map vm;
    try {
        po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);

        if (vm.count("help")) {
            std::cout << desc << "\n";
            exit(0);
        }

        po::notify(vm);

    } catch (std::exception &e) {
        cerr << "Error: " << e.what() << endl;
        return false;
    }

    return true;
}
<commit_msg>disambiguate boost::ref from new (C++11) std::ref<commit_after>/*******************************************************************************
 * Copyright (c) 2012, Dougal J. Sutherland (dsutherl@cs.cmu.edu).             *
 * All rights reserved.                                                        *
 *                                                                             *
 * Redistribution and use in source and binary forms, with or without          *
 * modification, are permitted provided that the following conditions are met: *
 *                                                                             *
 *     * Redistributions of source code must retain the above copyright        *
 *       notice, this list of conditions and the following disclaimer.         *
 *                                                                             *
 *     * Redistributions in binary form must reproduce the above copyright     *
 *       notice, this list of conditions and the following disclaimer in the   *
 *       documentation and/or other materials provided with the distribution.  *
 *                                                                             *
 *     * Neither the name of Carnegie Mellon University nor the                *
 *       names of the contributors may be used to endorse or promote products  *
 *       derived from this software without specific prior written permission. *
 *                                                                             *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" *
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE   *
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE  *
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE   *
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR         *
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF        *
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS    *
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN     *
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)     *
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE  *
 * POSSIBILITY OF SUCH DAMAGE.                                                 *
 ******************************************************************************/
#include "np-divs/np_divs.hpp"
#include "np-divs/matrix_io.hpp"
#include "np-divs/div-funcs/div_func.hpp"
#include "np-divs/div-funcs/from_str.hpp"

#include <iostream>
#include <fstream>
#include <string>

#include <boost/bind.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/program_options.hpp>
#include <boost/ptr_container/ptr_vector.hpp>
#include <boost/utility.hpp>

#include <flann/util/matrix.h>

using namespace npdivs;
using namespace std;
using namespace boost;
namespace po = boost::program_options;

struct ProgOpts : noncopyable {
    string x_bags_file;
    string y_bags_file;
    string results_file;

    ptr_vector<DivFunc> div_funcs;

    size_t k;
    size_t num_threads;

    flann::IndexParams index_params;
    flann::SearchParams search_params;

    size_t show_progress;

    void parse_div_funcs(const vector<string> &names) {
        for (size_t i = 0; i < names.size(); i++) {
            div_funcs.push_back(div_func_from_str(names[i]));
        }
    }

    void parse_index(const string name) {
        index_params = index_params_from_str(name);
    }
};

bool parse_args(int argc, char ** argv, ProgOpts& opts);

int main(int argc, char ** argv) {
    typedef flann::Matrix<double> Matrix;

    try {
        ProgOpts opts;
        opts.search_params = flann::SearchParams(64);
        if (!parse_args(argc, argv, opts))
            return 1;

        size_t num_df = opts.div_funcs.size();
        if (num_df == 0) {
            cerr << "Error: at least one div func is required\n";
            return 1;
        }

        // load input bags
        // TODO - gracefully handle nonexisting files
        size_t num_x;
        Matrix* x_bags;
        if (opts.x_bags_file == "-") {
            x_bags = matrices_from_csv(cin, num_x);
        } else {
            ifstream ifs(opts.x_bags_file.c_str(), ifstream::in);
            x_bags = matrices_from_csv(ifs, num_x);
            if (opts.show_progress)
                cerr << "Read " << num_x << " x bags.\n";
        }

        size_t num_y;
        Matrix* y_bags;
        if (opts.y_bags_file.empty()) {
            y_bags = NULL;
            num_y = num_x;
        } else if (opts.y_bags_file == "-") {
            y_bags = matrices_from_csv(cin, num_y);
        } else {
            ifstream ifs(opts.y_bags_file.c_str(), ifstream::in);
            y_bags = matrices_from_csv(ifs, num_y);
            if (opts.show_progress)
                cerr << "Read " << num_x << " y bags.\n";
        }

        Matrix* results = alloc_matrix_array<double>(num_df, num_x, num_y);

        DivParams params(opts.k, opts.index_params, opts.search_params,
                opts.num_threads, opts.show_progress);

        boost::posix_time::ptime t_start = boost::posix_time::second_clock::local_time();

        np_divs(x_bags, num_x, y_bags, num_y, opts.div_funcs, results, params);

        boost::posix_time::ptime t_end = boost::posix_time::second_clock::local_time();
        if (opts.show_progress)
            cerr << "Computation took " << (t_end - t_start).total_seconds() << " seconds.\n";

        if (opts.results_file == "-") {
            matrix_array_to_csv(cout, results, num_df);
        } else {
            ofstream ofs(opts.results_file.c_str());
            matrix_array_to_csv(ofs, results, num_df);
        }

        free_matrix_array(results, num_df);

    } catch (std::exception &e) {
        cerr << "Error: " << e.what() << endl;
        exit(1);
    }

    return 0;
}

// TODO nicer handling of matrix inputs
// TODO optionally support HDF5 inputs
// TODO positional arguments for x_bags, y_bags
// TODO support setting {index,search}_params
bool parse_args(int argc, char ** argv, ProgOpts& opts) {
    po::options_description desc("Allowed options");
    desc.add_options()
        ("help,h", "Produce this help message.")
        ("x-bags,x",
            po::value<string>(&opts.x_bags_file)->default_value("-"),
            "CSV-style file containing matrices separated by blank lines; "
            "- means stdin.")
        ("y-bags,y",
            po::value<string>(&opts.y_bags_file),
            "CSV-style file containing matrices separated by blank lines. "
            "If passed, divergences are calculated between the x bags (rows "
            "in the output matrix) and the y bags (columns); if not passed, "
            "divergences are calculated from the x bags to themselves. Pass "
            "- to use stdin; if both x and y are from stdin, x is read first "
            "and the two groups must be separated by exactly two blank lines. ")
        ("results,r",
            po::value<string>(&opts.results_file)->default_value("-"),
            "Where to write CSV-style matrix of results, where m[i,j] = "
            "div(x_i, y_j). Use - for stdout.")
        ("div-func,f",
            po::value< vector<string> >()->composing()
               ->notifier(bind(&ProgOpts::parse_div_funcs, boost::ref(opts), _1)),
            "Divergence functions to use; can be specified more than once. If "
            "none passed, uses L2. Format is name:arg1:arg2:..., where argN "
            "refers to the Nth argument to the corresponding DivFunc's "
            "constructor. Some support a first argument specifying a "
            "parameter: renyi:.99 means the Renyi-.99 divergence. All support "
            "a last parameter, which determines the way that large values are "
            "normalized: l2:.95 or renyi:.99:.95 means certain calculated "
            "intermediate values above the 95th percentile are cut down; 1 "
            "means not to do this; default is .99. All extra arguments are "
            "optional.")
        ("num-threads",
            po::value<size_t>(&opts.num_threads)->default_value(0),
            "Number of threads to use for calculations. 0 means one per core.")
        ("neighbors,k",
            po::value<size_t>(&opts.k)->default_value(3),
            "The k for k-nearest-neighbor calculations.")
        ("index,i",
            po::value<string>()->default_value("kdtree")
                ->notifier(bind(&ProgOpts::parse_index, boost::ref(opts), _1)),
            "The nearest-neighbor index to use. Options: linear, kdtree.")
        ("progress,p",
            po::value<size_t>(&opts.show_progress)->default_value(1000),
            "Show progress indications every X computations (default 1000; "
            "use 0 to not show any).")
    ;

    po::variables_map vm;
    try {
        po::store(po::command_line_parser(argc, argv).options(desc).run(), vm);

        if (vm.count("help")) {
            std::cout << desc << "\n";
            exit(0);
        }

        po::notify(vm);

    } catch (std::exception &e) {
        cerr << "Error: " << e.what() << endl;
        return false;
    }

    return true;
}
<|endoftext|>