<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/in_process_webkit/dom_storage_context.h"

#include <algorithm>

#include "base/file_path.h"
#include "base/file_util.h"
#include "base/string_util.h"
#include "chrome/browser/browser_thread.h"
#include "chrome/browser/in_process_webkit/dom_storage_area.h"
#include "chrome/browser/in_process_webkit/dom_storage_namespace.h"
#include "chrome/browser/in_process_webkit/webkit_context.h"
#include "chrome/common/dom_storage_common.h"
#include "third_party/WebKit/WebKit/chromium/public/WebSecurityOrigin.h"
#include "third_party/WebKit/WebKit/chromium/public/WebString.h"
#include "webkit/glue/webkit_glue.h"

const FilePath::CharType DOMStorageContext::kLocalStorageDirectory[] =
    FILE_PATH_LITERAL("Local Storage");

const FilePath::CharType DOMStorageContext::kLocalStorageExtension[] =
    FILE_PATH_LITERAL(".localstorage");

static const FilePath::CharType kLocalStorageOldPath[] =
    FILE_PATH_LITERAL("localStorage");

// TODO(jorlow): Remove after Chrome 4 ships.
static void MigrateLocalStorageDirectory(const FilePath& data_path) {
  FilePath new_path = data_path.Append(
      DOMStorageContext::kLocalStorageDirectory);
  FilePath old_path = data_path.Append(kLocalStorageOldPath);
  if (!file_util::DirectoryExists(new_path) &&
      file_util::DirectoryExists(old_path)) {
    file_util::Move(old_path, new_path);
  }
}

DOMStorageContext::DOMStorageContext(WebKitContext* webkit_context)
    : last_storage_area_id_(0),
      last_session_storage_namespace_id_on_ui_thread_(kLocalStorageNamespaceId),
      last_session_storage_namespace_id_on_io_thread_(kLocalStorageNamespaceId),
      webkit_context_(webkit_context) {
}

DOMStorageContext::~DOMStorageContext() {
  // This should not go away until all DOM Storage Dispatcher hosts have gone
  // away.  And they remove themselves from this list.
  DCHECK(dispatcher_host_set_.empty());

  for (StorageNamespaceMap::iterator iter(storage_namespace_map_.begin());
       iter != storage_namespace_map_.end(); ++iter) {
    delete iter->second;
  }
}

int64 DOMStorageContext::AllocateStorageAreaId() {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  return ++last_storage_area_id_;
}

int64 DOMStorageContext::AllocateSessionStorageNamespaceId() {
  if (BrowserThread::CurrentlyOn(BrowserThread::UI))
    return ++last_session_storage_namespace_id_on_ui_thread_;
  return --last_session_storage_namespace_id_on_io_thread_;
}

int64 DOMStorageContext::CloneSessionStorage(int64 original_id) {
  DCHECK(!BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 clone_id = AllocateSessionStorageNamespaceId();
  BrowserThread::PostTask(
      BrowserThread::WEBKIT, FROM_HERE, NewRunnableFunction(
          &DOMStorageContext::CompleteCloningSessionStorage,
          this, original_id, clone_id));
  return clone_id;
}

void DOMStorageContext::RegisterStorageArea(DOMStorageArea* storage_area) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_area->id();
  DCHECK(!GetStorageArea(id));
  storage_area_map_[id] = storage_area;
}

void DOMStorageContext::UnregisterStorageArea(DOMStorageArea* storage_area) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_area->id();
  DCHECK(GetStorageArea(id));
  storage_area_map_.erase(id);
}

DOMStorageArea* DOMStorageContext::GetStorageArea(int64 id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageAreaMap::iterator iter = storage_area_map_.find(id);
  if (iter == storage_area_map_.end())
    return NULL;
  return iter->second;
}

void DOMStorageContext::DeleteSessionStorageNamespace(int64 namespace_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageNamespaceMap::iterator iter =
      storage_namespace_map_.find(namespace_id);
  if (iter == storage_namespace_map_.end())
    return;
  DCHECK(iter->second->dom_storage_type() == DOM_STORAGE_SESSION);
  delete iter->second;
  storage_namespace_map_.erase(iter);
}

DOMStorageNamespace* DOMStorageContext::GetStorageNamespace(
    int64 id, bool allocation_allowed) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageNamespaceMap::iterator iter = storage_namespace_map_.find(id);
  if (iter != storage_namespace_map_.end())
    return iter->second;
  if (!allocation_allowed)
    return NULL;
  if (id == kLocalStorageNamespaceId)
    return CreateLocalStorage();
  return CreateSessionStorage(id);
}

void DOMStorageContext::RegisterDispatcherHost(
    DOMStorageDispatcherHost* dispatcher_host) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  DCHECK(dispatcher_host_set_.find(dispatcher_host) ==
         dispatcher_host_set_.end());
  dispatcher_host_set_.insert(dispatcher_host);
}

void DOMStorageContext::UnregisterDispatcherHost(
    DOMStorageDispatcherHost* dispatcher_host) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  DCHECK(dispatcher_host_set_.find(dispatcher_host) !=
         dispatcher_host_set_.end());
  dispatcher_host_set_.erase(dispatcher_host);
}

const DOMStorageContext::DispatcherHostSet*
DOMStorageContext::GetDispatcherHostSet() const {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  return &dispatcher_host_set_;
}

void DOMStorageContext::PurgeMemory() {
  // It is only safe to purge the memory from the LocalStorage namespace,
  // because it is backed by disk and can be reloaded later.  If we purge a
  // SessionStorage namespace, its data will be gone forever, because it isn't
  // currently backed by disk.
  DOMStorageNamespace* local_storage =
      GetStorageNamespace(kLocalStorageNamespaceId, false);
  if (local_storage)
    local_storage->PurgeMemory();
}

void DOMStorageContext::DeleteDataModifiedSince(
    const base::Time& cutoff,
    const char* url_scheme_to_be_skipped,
    const std::vector<string16>& protected_origins) {
  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  PurgeMemory();

  file_util::FileEnumerator file_enumerator(
      webkit_context_->data_path().Append(kLocalStorageDirectory), false,
      file_util::FileEnumerator::FILES);
  for (FilePath path = file_enumerator.Next(); !path.value().empty();
       path = file_enumerator.Next()) {
    WebKit::WebSecurityOrigin web_security_origin =
        WebKit::WebSecurityOrigin::createFromDatabaseIdentifier(
            webkit_glue::FilePathToWebString(path.BaseName()));
    if (EqualsASCII(web_security_origin.protocol(), url_scheme_to_be_skipped))
      continue;

    std::vector<string16>::const_iterator find_iter =
        std::find(protected_origins.begin(), protected_origins.end(),
            web_security_origin.databaseIdentifier());
    if (find_iter != protected_origins.end())
      continue;

    file_util::FileEnumerator::FindInfo find_info;
    file_enumerator.GetFindInfo(&find_info);
    if (file_util::HasFileBeenModifiedSince(find_info, cutoff))
      file_util::Delete(path, false);
  }
}

void DOMStorageContext::DeleteLocalStorageFile(const FilePath& file_path) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));

  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  // TODO(bulach): both this method and DeleteDataModifiedSince could purge
  // only the memory used by the specific file instead of all memory at once.
  // See http://crbug.com/32000
  PurgeMemory();
  file_util::Delete(file_path, false);
}

void DOMStorageContext::DeleteLocalStorageForOrigin(const string16& origin_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  DeleteLocalStorageFile(GetLocalStorageFilePath(origin_id));
}

void DOMStorageContext::DeleteAllLocalStorageFiles() {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));

  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  PurgeMemory();

  file_util::FileEnumerator file_enumerator(
      webkit_context_->data_path().Append(kLocalStorageDirectory), false,
      file_util::FileEnumerator::FILES);
  for (FilePath file_path = file_enumerator.Next(); !file_path.empty();
       file_path = file_enumerator.Next()) {
    if (file_path.Extension() == kLocalStorageExtension)
      file_util::Delete(file_path, false);
  }
}

DOMStorageNamespace* DOMStorageContext::CreateLocalStorage() {
  FilePath data_path = webkit_context_->data_path();
  FilePath dir_path;
  if (!data_path.empty()) {
    MigrateLocalStorageDirectory(data_path);
    dir_path = data_path.Append(kLocalStorageDirectory);
  }
  DOMStorageNamespace* new_namespace =
      DOMStorageNamespace::CreateLocalStorageNamespace(this, dir_path);
  RegisterStorageNamespace(new_namespace);
  return new_namespace;
}

DOMStorageNamespace* DOMStorageContext::CreateSessionStorage(
    int64 namespace_id) {
  DOMStorageNamespace* new_namespace =
      DOMStorageNamespace::CreateSessionStorageNamespace(this, namespace_id);
  RegisterStorageNamespace(new_namespace);
  return new_namespace;
}

void DOMStorageContext::RegisterStorageNamespace(
    DOMStorageNamespace* storage_namespace) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_namespace->id();
  DCHECK(!GetStorageNamespace(id, false));
  storage_namespace_map_[id] = storage_namespace;
}

/* static */
void DOMStorageContext::CompleteCloningSessionStorage(
    DOMStorageContext* context, int64 existing_id, int64 clone_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  DOMStorageNamespace* existing_namespace =
      context->GetStorageNamespace(existing_id, false);
  // If nothing exists, then there's nothing to clone.
  if (existing_namespace)
    context->RegisterStorageNamespace(existing_namespace->Copy(clone_id));
}

// static
void DOMStorageContext::ClearLocalState(const FilePath& profile_path,
                                        const char* url_scheme_to_be_skipped) {
  file_util::FileEnumerator file_enumerator(profile_path.Append(
      kLocalStorageDirectory), false, file_util::FileEnumerator::FILES);
  for (FilePath file_path = file_enumerator.Next(); !file_path.empty();
       file_path = file_enumerator.Next()) {
    if (file_path.Extension() == kLocalStorageExtension) {
      WebKit::WebSecurityOrigin web_security_origin =
          WebKit::WebSecurityOrigin::createFromDatabaseIdentifier(
              webkit_glue::FilePathToWebString(file_path.BaseName()));
      if (!EqualsASCII(web_security_origin.protocol(),
                       url_scheme_to_be_skipped))
        file_util::Delete(file_path, false);
    }
  }
}

FilePath DOMStorageContext::GetLocalStorageFilePath(
    const string16& origin_id) const {
  FilePath storageDir = webkit_context_->data_path().Append(
      DOMStorageContext::kLocalStorageDirectory);
  FilePath::StringType id =
      webkit_glue::WebStringToFilePathString(origin_id);
  return storageDir.Append(id.append(kLocalStorageExtension));
}
<commit_msg>Coding style clean up of DomStorageContext.<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/in_process_webkit/dom_storage_context.h"

#include <algorithm>

#include "base/file_path.h"
#include "base/file_util.h"
#include "base/string_util.h"
#include "chrome/browser/browser_thread.h"
#include "chrome/browser/in_process_webkit/dom_storage_area.h"
#include "chrome/browser/in_process_webkit/dom_storage_namespace.h"
#include "chrome/browser/in_process_webkit/webkit_context.h"
#include "chrome/common/dom_storage_common.h"
#include "third_party/WebKit/WebKit/chromium/public/WebSecurityOrigin.h"
#include "third_party/WebKit/WebKit/chromium/public/WebString.h"
#include "webkit/glue/webkit_glue.h"

using WebKit::WebSecurityOrigin;

const FilePath::CharType DOMStorageContext::kLocalStorageDirectory[] =
    FILE_PATH_LITERAL("Local Storage");

const FilePath::CharType DOMStorageContext::kLocalStorageExtension[] =
    FILE_PATH_LITERAL(".localstorage");

static const FilePath::CharType kLocalStorageOldPath[] =
    FILE_PATH_LITERAL("localStorage");

// TODO(jorlow): Remove after Chrome 4 ships.
static void MigrateLocalStorageDirectory(const FilePath& data_path) {
  FilePath new_path = data_path.Append(
      DOMStorageContext::kLocalStorageDirectory);
  FilePath old_path = data_path.Append(kLocalStorageOldPath);
  if (!file_util::DirectoryExists(new_path) &&
      file_util::DirectoryExists(old_path)) {
    file_util::Move(old_path, new_path);
  }
}

DOMStorageContext::DOMStorageContext(WebKitContext* webkit_context)
    : last_storage_area_id_(0),
      last_session_storage_namespace_id_on_ui_thread_(kLocalStorageNamespaceId),
      last_session_storage_namespace_id_on_io_thread_(kLocalStorageNamespaceId),
      webkit_context_(webkit_context) {
}

DOMStorageContext::~DOMStorageContext() {
  // This should not go away until all DOM Storage Dispatcher hosts have gone
  // away.  And they remove themselves from this list.
  DCHECK(dispatcher_host_set_.empty());

  for (StorageNamespaceMap::iterator iter(storage_namespace_map_.begin());
       iter != storage_namespace_map_.end(); ++iter) {
    delete iter->second;
  }
}

int64 DOMStorageContext::AllocateStorageAreaId() {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  return ++last_storage_area_id_;
}

int64 DOMStorageContext::AllocateSessionStorageNamespaceId() {
  if (BrowserThread::CurrentlyOn(BrowserThread::UI))
    return ++last_session_storage_namespace_id_on_ui_thread_;
  return --last_session_storage_namespace_id_on_io_thread_;
}

int64 DOMStorageContext::CloneSessionStorage(int64 original_id) {
  DCHECK(!BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 clone_id = AllocateSessionStorageNamespaceId();
  BrowserThread::PostTask(
      BrowserThread::WEBKIT, FROM_HERE, NewRunnableFunction(
          &DOMStorageContext::CompleteCloningSessionStorage,
          this, original_id, clone_id));
  return clone_id;
}

void DOMStorageContext::RegisterStorageArea(DOMStorageArea* storage_area) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_area->id();
  DCHECK(!GetStorageArea(id));
  storage_area_map_[id] = storage_area;
}

void DOMStorageContext::UnregisterStorageArea(DOMStorageArea* storage_area) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_area->id();
  DCHECK(GetStorageArea(id));
  storage_area_map_.erase(id);
}

DOMStorageArea* DOMStorageContext::GetStorageArea(int64 id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageAreaMap::iterator iter = storage_area_map_.find(id);
  if (iter == storage_area_map_.end())
    return NULL;
  return iter->second;
}

void DOMStorageContext::DeleteSessionStorageNamespace(int64 namespace_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageNamespaceMap::iterator iter =
      storage_namespace_map_.find(namespace_id);
  if (iter == storage_namespace_map_.end())
    return;
  DCHECK(iter->second->dom_storage_type() == DOM_STORAGE_SESSION);
  delete iter->second;
  storage_namespace_map_.erase(iter);
}

DOMStorageNamespace* DOMStorageContext::GetStorageNamespace(
    int64 id, bool allocation_allowed) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  StorageNamespaceMap::iterator iter = storage_namespace_map_.find(id);
  if (iter != storage_namespace_map_.end())
    return iter->second;
  if (!allocation_allowed)
    return NULL;
  if (id == kLocalStorageNamespaceId)
    return CreateLocalStorage();
  return CreateSessionStorage(id);
}

void DOMStorageContext::RegisterDispatcherHost(
    DOMStorageDispatcherHost* dispatcher_host) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  DCHECK(dispatcher_host_set_.find(dispatcher_host) ==
         dispatcher_host_set_.end());
  dispatcher_host_set_.insert(dispatcher_host);
}

void DOMStorageContext::UnregisterDispatcherHost(
    DOMStorageDispatcherHost* dispatcher_host) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  DCHECK(dispatcher_host_set_.find(dispatcher_host) !=
         dispatcher_host_set_.end());
  dispatcher_host_set_.erase(dispatcher_host);
}

const DOMStorageContext::DispatcherHostSet*
DOMStorageContext::GetDispatcherHostSet() const {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
  return &dispatcher_host_set_;
}

void DOMStorageContext::PurgeMemory() {
  // It is only safe to purge the memory from the LocalStorage namespace,
  // because it is backed by disk and can be reloaded later.  If we purge a
  // SessionStorage namespace, its data will be gone forever, because it isn't
  // currently backed by disk.
  DOMStorageNamespace* local_storage =
      GetStorageNamespace(kLocalStorageNamespaceId, false);
  if (local_storage)
    local_storage->PurgeMemory();
}

void DOMStorageContext::DeleteDataModifiedSince(
    const base::Time& cutoff,
    const char* url_scheme_to_be_skipped,
    const std::vector<string16>& protected_origins) {
  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  PurgeMemory();

  file_util::FileEnumerator file_enumerator(
      webkit_context_->data_path().Append(kLocalStorageDirectory), false,
      file_util::FileEnumerator::FILES);
  for (FilePath path = file_enumerator.Next(); !path.value().empty();
       path = file_enumerator.Next()) {
    WebSecurityOrigin web_security_origin =
        WebSecurityOrigin::createFromDatabaseIdentifier(
            webkit_glue::FilePathToWebString(path.BaseName()));
    if (EqualsASCII(web_security_origin.protocol(), url_scheme_to_be_skipped))
      continue;

    std::vector<string16>::const_iterator find_iter =
        std::find(protected_origins.begin(), protected_origins.end(),
            web_security_origin.databaseIdentifier());
    if (find_iter != protected_origins.end())
      continue;

    file_util::FileEnumerator::FindInfo find_info;
    file_enumerator.GetFindInfo(&find_info);
    if (file_util::HasFileBeenModifiedSince(find_info, cutoff))
      file_util::Delete(path, false);
  }
}

void DOMStorageContext::DeleteLocalStorageFile(const FilePath& file_path) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));

  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  // TODO(bulach): both this method and DeleteDataModifiedSince could purge
  // only the memory used by the specific file instead of all memory at once.
  // See http://crbug.com/32000
  PurgeMemory();
  file_util::Delete(file_path, false);
}

void DOMStorageContext::DeleteLocalStorageForOrigin(const string16& origin_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  DeleteLocalStorageFile(GetLocalStorageFilePath(origin_id));
}

void DOMStorageContext::DeleteAllLocalStorageFiles() {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));

  // Make sure that we don't delete a database that's currently being accessed
  // by unloading all of the databases temporarily.
  PurgeMemory();

  file_util::FileEnumerator file_enumerator(
      webkit_context_->data_path().Append(kLocalStorageDirectory), false,
      file_util::FileEnumerator::FILES);
  for (FilePath file_path = file_enumerator.Next(); !file_path.empty();
       file_path = file_enumerator.Next()) {
    if (file_path.Extension() == kLocalStorageExtension)
      file_util::Delete(file_path, false);
  }
}

DOMStorageNamespace* DOMStorageContext::CreateLocalStorage() {
  FilePath data_path = webkit_context_->data_path();
  FilePath dir_path;
  if (!data_path.empty()) {
    MigrateLocalStorageDirectory(data_path);
    dir_path = data_path.Append(kLocalStorageDirectory);
  }
  DOMStorageNamespace* new_namespace =
      DOMStorageNamespace::CreateLocalStorageNamespace(this, dir_path);
  RegisterStorageNamespace(new_namespace);
  return new_namespace;
}

DOMStorageNamespace* DOMStorageContext::CreateSessionStorage(
    int64 namespace_id) {
  DOMStorageNamespace* new_namespace =
      DOMStorageNamespace::CreateSessionStorageNamespace(this, namespace_id);
  RegisterStorageNamespace(new_namespace);
  return new_namespace;
}

void DOMStorageContext::RegisterStorageNamespace(
    DOMStorageNamespace* storage_namespace) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  int64 id = storage_namespace->id();
  DCHECK(!GetStorageNamespace(id, false));
  storage_namespace_map_[id] = storage_namespace;
}

/* static */
void DOMStorageContext::CompleteCloningSessionStorage(
    DOMStorageContext* context, int64 existing_id, int64 clone_id) {
  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::WEBKIT));
  DOMStorageNamespace* existing_namespace =
      context->GetStorageNamespace(existing_id, false);
  // If nothing exists, then there's nothing to clone.
  if (existing_namespace)
    context->RegisterStorageNamespace(existing_namespace->Copy(clone_id));
}

// static
void DOMStorageContext::ClearLocalState(const FilePath& profile_path,
                                        const char* url_scheme_to_be_skipped) {
  file_util::FileEnumerator file_enumerator(profile_path.Append(
      kLocalStorageDirectory), false, file_util::FileEnumerator::FILES);
  for (FilePath file_path = file_enumerator.Next(); !file_path.empty();
       file_path = file_enumerator.Next()) {
    if (file_path.Extension() == kLocalStorageExtension) {
      WebSecurityOrigin web_security_origin =
          WebSecurityOrigin::createFromDatabaseIdentifier(
              webkit_glue::FilePathToWebString(file_path.BaseName()));
      if (!EqualsASCII(web_security_origin.protocol(),
                       url_scheme_to_be_skipped)) {
        file_util::Delete(file_path, false);
      }
    }
  }
}

FilePath DOMStorageContext::GetLocalStorageFilePath(
    const string16& origin_id) const {
  FilePath storageDir = webkit_context_->data_path().Append(
      DOMStorageContext::kLocalStorageDirectory);
  FilePath::StringType id =
      webkit_glue::WebStringToFilePathString(origin_id);
  return storageDir.Append(id.append(kLocalStorageExtension));
}
<|endoftext|>


<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/renderer/automation/dom_automation_controller.h"

#include "chrome/common/json_value_serializer.h"
#include "chrome/common/render_messages.h"
#include "base/string_util.h"

IPC::Message::Sender* DomAutomationController::sender_(NULL);
int DomAutomationController::routing_id_(MSG_ROUTING_NONE);
int DomAutomationController::automation_id_(MSG_ROUTING_NONE);

DomAutomationController::DomAutomationController(){
  BindMethod("send", &DomAutomationController::send);
  BindMethod("setAutomationId", &DomAutomationController::setAutomationId);
}

void DomAutomationController::send(const CppArgumentList& args,
                                   CppVariant* result) {
  if (args.size() != 1) {
    result->SetNull();
    return;
  }

  if (automation_id_ == MSG_ROUTING_NONE) {
    result->SetNull();
    return;
  }

  std::string json;
  JSONStringValueSerializer serializer(&json);
  Value* value = NULL;

  // Warning: note that JSON officially requires the root-level object to be
  // an object (e.g. {foo:3}) or an array, while here we're serializing
  // strings, bools, etc. to "JSON".  This only works because (a) the JSON
  // writer is lenient, and (b) on the receiving side we wrap the JSON string
  // in square brackets, converting it to an array, then parsing it and
  // grabbing the 0th element to get the value out.
  switch(args[0].type) {
    case NPVariantType_String: {
      value = Value::CreateStringValue(UTF8ToWide(args[0].ToString()));
      break;
    }
    case NPVariantType_Bool: {
      value = Value::CreateBooleanValue(args[0].ToBoolean());
      break;
    }
    case NPVariantType_Int32: {
      value = Value::CreateIntegerValue(args[0].ToInt32());
      break;
    }
    case NPVariantType_Double: {
      // The value that is sent back is an integer while it is treated
      // as a double in this binding. The reason being that KJS treats
      // any number value as a double. Refer for more details,
      // chrome/third_party/webkit/src/JavaScriptCore/bindings/c/c_utility.cpp
      value = Value::CreateIntegerValue(args[0].ToInt32());
      break;
    }
    default: {
      result->SetNull();
      return;
    }
  }

  bool succeeded = serializer.Serialize(*value);
  if (!succeeded) {
    result->SetNull();
    return;
  }

  succeeded = sender_->Send(
    new ViewHostMsg_DomOperationResponse(routing_id_, json, automation_id_));

  automation_id_ = MSG_ROUTING_NONE;

  result->Set(succeeded);
  return;
}

void DomAutomationController::setAutomationId(
    const CppArgumentList& args, CppVariant* result) {
  if (args.size() != 1) {
    result->SetNull();
    return;
  }

  // The check here is for NumberType and not Int32 as
  // KJS::JSType only defines a NumberType (no Int32)
  if (!args[0].isNumber()) {
    result->SetNull();
    return;
  }

  automation_id_ = args[0].ToInt32();
  result->Set(true);
}

<commit_msg>Changed DomAutomationController to use UTF-8 strings.<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/renderer/automation/dom_automation_controller.h"

#include "chrome/common/json_value_serializer.h"
#include "chrome/common/render_messages.h"
#include "base/string_util.h"

IPC::Message::Sender* DomAutomationController::sender_(NULL);
int DomAutomationController::routing_id_(MSG_ROUTING_NONE);
int DomAutomationController::automation_id_(MSG_ROUTING_NONE);

DomAutomationController::DomAutomationController(){
  BindMethod("send", &DomAutomationController::send);
  BindMethod("setAutomationId", &DomAutomationController::setAutomationId);
}

void DomAutomationController::send(const CppArgumentList& args,
                                   CppVariant* result) {
  if (args.size() != 1) {
    result->SetNull();
    return;
  }

  if (automation_id_ == MSG_ROUTING_NONE) {
    result->SetNull();
    return;
  }

  std::string json;
  JSONStringValueSerializer serializer(&json);
  Value* value = NULL;

  // Warning: note that JSON officially requires the root-level object to be
  // an object (e.g. {foo:3}) or an array, while here we're serializing
  // strings, bools, etc. to "JSON".  This only works because (a) the JSON
  // writer is lenient, and (b) on the receiving side we wrap the JSON string
  // in square brackets, converting it to an array, then parsing it and
  // grabbing the 0th element to get the value out.
  switch(args[0].type) {
    case NPVariantType_String: {
      value = Value::CreateStringValue(args[0].ToString());
      break;
    }
    case NPVariantType_Bool: {
      value = Value::CreateBooleanValue(args[0].ToBoolean());
      break;
    }
    case NPVariantType_Int32: {
      value = Value::CreateIntegerValue(args[0].ToInt32());
      break;
    }
    case NPVariantType_Double: {
      // The value that is sent back is an integer while it is treated
      // as a double in this binding. The reason being that KJS treats
      // any number value as a double. Refer for more details,
      // chrome/third_party/webkit/src/JavaScriptCore/bindings/c/c_utility.cpp
      value = Value::CreateIntegerValue(args[0].ToInt32());
      break;
    }
    default: {
      result->SetNull();
      return;
    }
  }

  bool succeeded = serializer.Serialize(*value);
  if (!succeeded) {
    result->SetNull();
    return;
  }

  succeeded = sender_->Send(
    new ViewHostMsg_DomOperationResponse(routing_id_, json, automation_id_));

  automation_id_ = MSG_ROUTING_NONE;

  result->Set(succeeded);
  return;
}

void DomAutomationController::setAutomationId(
    const CppArgumentList& args, CppVariant* result) {
  if (args.size() != 1) {
    result->SetNull();
    return;
  }

  // The check here is for NumberType and not Int32 as
  // KJS::JSType only defines a NumberType (no Int32)
  if (!args[0].isNumber()) {
    result->SetNull();
    return;
  }

  automation_id_ = args[0].ToInt32();
  result->Set(true);
}

<|endoftext|>


<commit_before>#include <iostream>
#include "DmpLoadParameters.h"
#include <fstream>
#include <vector>
#include "TMath.h"
#include "TCanvas.h"
#include "TH2F.h"
#include "TStyle.h"

void Split(string fname)
{
   DmpParameterSteering  steering;
   DmpParameterHolder  rel_par;
   DAMPE::LoadParameters(fname,rel_par,steering);
   string gfName = "good_"+fname;
   ofstream goodF(gfName.c_str(),std::ios::out);
   string ffName = "failed_"+fname;
   ofstream failedF(ffName.c_str(),std::ios::out);

   for(DmpParameterHolder::iterator it = rel_par.begin();it != rel_par.end();++it){
     if(it->second.at(8) == -1){
       failedF<<it->first;
       for(size_t i=0;i < it->second.size();++i){
         failedF<<"\t\t"<<it->second.at(i);
       }
       failedF<<std::endl;
     }else{
       goodF<<it->first;
       for(size_t i=0;i < it->second.size();++i){
         goodF<<"\t\t"<<it->second.at(i);
       }
       goodF<<std::endl;
     }
   }
}

void FindBest(std::vector<std::string> filenames,string outPutFileName="allGood.txt")
{
  int nFiles = filenames.size();
  std::vector<DmpParameterHolder>  pars(nFiles);
  DmpParameterSteering  steering;
  for(int i=0;i<nFiles;++i){
    DAMPE::LoadParameters(filenames[i],pars.at(i),steering);
  }
  ofstream goodF(outPutFileName.c_str(),std::ios::out);
  for(DmpParameterHolder::iterator it = pars[0].begin();it!=pars[0].end();++it){
    for(int i=0;i<nFiles;++i){
      if(pars[i][it->first].at(8) == -1) continue;
      if(TMath::Abs(pars[i][it->first].at(4)) > 40) continue;
      if(pars[i][it->first].at(5) == -999) continue;
      if(pars[i][it->first].at(6) == -999) continue;
      if(it->second.at(8) == -1){
        it->second = pars[i][it->first];
        continue;
      }
      if(it->second.at(6) == -999 || it->second.at(6) == 999){
        it->second = pars[i][it->first];
        continue;
      }
      bool comp0 = TMath::Abs(pars[i][it->first].at(4)) < TMath::Abs(it->second.at(4)); // p0
      bool comp1 = pars[i][it->first].at(5) < it->second.at(5); // p1
      double deltchi = TMath::Abs(TMath::Abs(pars[i][it->first].at(6))-1) - TMath::Abs(TMath::Abs(it->second.at(6)) - 1); // chi2
      bool  comp2 = deltchi < 0? 1: 0;
      bool comp3 = pars[i][it->first].at(7) > 0.666 * it->second.at(7); // entries
      bool comp4 = (pars[i][it->first].at(7) > 3*it->second.at(7)) && (deltchi < 1.5);
      if((comp0 && comp1 && comp3) || (comp0 && comp2 && comp3) || (comp1 && comp2 && comp3) || comp4){
        it->second = pars[i][it->first];
      }
    }
    goodF<<it->first;
    for(size_t n=0;n<it->second.size();++n){
      goodF<<"\t\t"<<it->second.at(n);
    }
    goodF<<std::endl;
  }
}

void Static(string fname)
{
  DmpParameterSteering  steering;
  DmpParameterHolder  rel_par;
  DAMPE::LoadParameters((std::string)fname,rel_par,steering);
  TString fn = fname;
  fn.Remove(fn.Last('.'));
  fname = fn+" p0 VS p1";
  TH2F *h2_p0p1 = new TH2F(fname.c_str(),"p0 VS p1;p1;p0",400,0,0.04,1200,-60,60);
  fname = fn+" chi2";
  TH2F *map_chi2 = new TH2F(fname.c_str(),"chi2;bar;layer",22,0,22,14,0,14);
  fname = fn+" p0";
  TH2F *map_p0 = new TH2F(fname.c_str(),"p0;bar;layer",22,0,22,14,0,14);
  fname = fn+" p1";
  TH2F *map_p1 = new TH2F(fname.c_str(),"p1;bar;layer",22,0,22,14,0,14);
  TString isPSD = fn;
  isPSD.ToUpper();
  if(isPSD.Contains("PSD")){
    delete map_chi2;
    fname = fn+" chi2";
    map_chi2 = new TH2F(fname.c_str(),"chi2;bar;layer",42,0,42,2,0,2);
    delete map_p0;
    fname = fn+" p0";
    map_p0 = new TH2F(fname.c_str(),"p0;bar;layer",42,0,42,2,0,2);
    delete map_p1;
    fname = fn+" p1";
    map_p1 = new TH2F(fname.c_str(),"p1;bar;layer",42,0,42,2,0,2);
  }

  for(DmpParameterHolder::iterator it = rel_par.begin();it != rel_par.end();++it){
    h2_p0p1->Fill(it->second.at(5),it->second.at(4));
    map_p0->Fill(it->second.at(1),it->second.at(0),it->second.at(4));
    map_p1->Fill(it->second.at(1),it->second.at(0),it->second.at(5));
    map_chi2->Fill(it->second.at(1),it->second.at(0),it->second.at(6));
  }

  gStyle->SetOptStat(111111);
  TCanvas *c1 = new TCanvas(fn,fn,600,800);
  c1->Divide(2,2);
  c1->cd(1);
  gPad->SetGrid();
  h2_p0p1->SetMarkerStyle(23);
  h2_p0p1->SetMarkerSize(0.8);
  h2_p0p1->Draw();
  c1->cd(2);
  map_chi2->Draw("colz");
  c1->cd(3);
  map_p0->Draw("colz");
  c1->cd(4);
  map_p1->Draw("colz");
}

<commit_msg>modified:   RelationMerge.C<commit_after>#include <iostream>
#include "DmpLoadParameters.h"
#include <fstream>
#include <vector>
#include "TMath.h"
#include "TCanvas.h"
#include "TH2F.h"
#include "TStyle.h"

void Split(string fname)
{
   DmpParameterSteering  steering;
   DmpParameterHolder  rel_par;
   DAMPE::LoadParameters(fname,rel_par,steering);
   string gfName = "good_"+fname;
   ofstream goodF(gfName.c_str(),std::ios::out);
   string ffName = "failed_"+fname;
   ofstream failedF(ffName.c_str(),std::ios::out);

   for(DmpParameterHolder::iterator it = rel_par.begin();it != rel_par.end();++it){
     if(it->second.at(4) == -1){
       failedF<<it->first;
       for(size_t i=0;i < it->second.size();++i){
         failedF<<"\t\t"<<it->second.at(i);
       }
       failedF<<std::endl;
     }else{
       goodF<<it->first;
       for(size_t i=0;i < it->second.size();++i){
         goodF<<"\t\t"<<it->second.at(i);
       }
       goodF<<std::endl;
     }
   }
}

void FindBest(std::vector<std::string> filenames,string outPutFileName="allGood.txt")
{
  int nFiles = filenames.size();
  std::vector<DmpParameterHolder>  pars(nFiles);
  DmpParameterSteering  steering;
  for(int i=0;i<nFiles;++i){
    DAMPE::LoadParameters(filenames[i],pars.at(i),steering);
  }
  ofstream goodF(outPutFileName.c_str(),std::ios::out);
  for(DmpParameterHolder::iterator it = pars[0].begin();it!=pars[0].end();++it){
    for(int i=0;i<nFiles;++i){
      if(pars[i][it->first].at(4) == -1) continue;
      if(TMath::Abs(pars[i][it->first].at(0)) > 40) continue;
      if(pars[i][it->first].at(1) == -999) continue;
      if(pars[i][it->first].at(2) == -999) continue;
      if(it->second.at(4) == -1){
        it->second = pars[i][it->first];
        continue;
      }
      if(it->second.at(2) == -999 || it->second.at(2) == 999){
        it->second = pars[i][it->first];
        continue;
      }
      bool comp0 = TMath::Abs(pars[i][it->first].at(0)) < TMath::Abs(it->second.at(0)); // p0
      bool comp1 = pars[i][it->first].at(1) < it->second.at(1); // p1
      double deltchi = TMath::Abs(TMath::Abs(pars[i][it->first].at(2))-1) - TMath::Abs(TMath::Abs(it->second.at(2)) - 1); // chi2
      bool  comp2 = deltchi < 0? 1: 0;
      bool comp3 = pars[i][it->first].at(3) > 0.666 * it->second.at(3); // entries
      bool comp4 = (pars[i][it->first].at(3) > 3*it->second.at(3)) && (deltchi < 1.5);
      if((comp0 && comp1 && comp3) || (comp0 && comp2 && comp3) || (comp1 && comp2 && comp3) || comp4){
        it->second = pars[i][it->first];
      }
    }
    goodF<<it->first;
    for(size_t n=0;n<it->second.size();++n){
      goodF<<"\t\t"<<it->second.at(n);
    }
    goodF<<std::endl;
  }
}

void Static(string fname)
{
  DmpParameterSteering  steering;
  DmpParameterHolder  rel_par;
  DAMPE::LoadParameters((std::string)fname,rel_par,steering);
  TString fn = fname;
  fn.Remove(fn.Last('.'));
  fname = fn+" p0 VS p1";
  TH2F *h2_p0p1 = new TH2F(fname.c_str(),"p0 VS p1;p1;p0",400,0,0.04,1200,-60,60);
  fname = fn+" chi2";
  TH2F *map_chi2 = new TH2F(fname.c_str(),"chi2;bar;layer",22,0,22,14,0,14);
  fname = fn+" p0";
  TH2F *map_p0 = new TH2F(fname.c_str(),"p0;bar;layer",22,0,22,14,0,14);
  fname = fn+" p1";
  TH2F *map_p1 = new TH2F(fname.c_str(),"p1;bar;layer",22,0,22,14,0,14);
  TString isPSD = fn;
  isPSD.ToUpper();
  if(isPSD.Contains("PSD")){
    delete map_chi2;
    fname = fn+" chi2";
    map_chi2 = new TH2F(fname.c_str(),"chi2;bar;layer",42,0,42,2,0,2);
    delete map_p0;
    fname = fn+" p0";
    map_p0 = new TH2F(fname.c_str(),"p0;bar;layer",42,0,42,2,0,2);
    delete map_p1;
    fname = fn+" p1";
    map_p1 = new TH2F(fname.c_str(),"p1;bar;layer",42,0,42,2,0,2);
  }

  for(DmpParameterHolder::iterator it = rel_par.begin();it != rel_par.end();++it){
    h2_p0p1->Fill(it->second.at(1),it->second.at(0));
    short l = DmpBgoBase::GetLayerID(it->second.at(0));
    short b = DmpBgoBase::GetLayerID(it->second.at(0));
    map_p0->Fill(b,l,it->second.at(0));
    map_p1->Fill(b,l,it->second.at(1));
    map_chi2->Fill(b,l,it->second.at(2));
  }

  gStyle->SetOptStat(111111);
  TCanvas *c1 = new TCanvas(fn,fn,600,800);
  c1->Divide(2,2);
  c1->cd(1);
  gPad->SetGrid();
  h2_p0p1->SetMarkerStyle(23);
  h2_p0p1->SetMarkerSize(0.8);
  h2_p0p1->Draw();
  c1->cd(2);
  map_chi2->Draw("colz");
  c1->cd(3);
  map_p0->Draw("colz");
  c1->cd(4);
  map_p1->Draw("colz");
}

<|endoftext|>


<commit_before>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "../common/harness.hh"
#include "lexer.hh"

TADPOLE_TEST(TadpoleLexer) {
  using TK = tadpole::TokenKind;
#define TESTEQ(k) TADPOLE_CHECK_EQ(lex.next_token().kind(), k)
#define DUMPLEX() do {\
    for (;;) {\
      auto t = lex.next_token();\
      std::fprintf(stdout, "%-24s | %-32s | %d\n",\
          tadpole::get_kind_name(t.kind()),\
          t.as_cstring(),\
          t.lineno());\
      if (t.kind() == TK::TK_EOF)\
        break;\
    }\
  } while (false)

  {
    std::string s =
      "print(\"Welcome to Tadpole !!!\");\nvar v = 34 * 56 / 22.45 + (88.76 - 27.4);";
    tadpole::Lexer lex(s);
    DUMPLEX();
  }

  {
    std::string s = "var a = 56;";
    tadpole::Lexer lex(s);
    TESTEQ(TK::KW_VAR);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_EQ);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_EOF);
  }

  {
    std::string s = "print(\"23 + 45 =\", 23 + 45);";
    tadpole::Lexer lex(s);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_STRING);
    TESTEQ(TK::TK_COMMA);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_PLUS);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_EOF);
  }

  {
    std::string s = "fn show_msg(msg) { print(msg); }";
    tadpole::Lexer lex(s);
    TESTEQ(TK::KW_FN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_LBRACE);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_RBRACE);
    TESTEQ(TK::TK_EOF);
  }

#undef DUMPLEX
#undef TESTEQ
}
<commit_msg>:white_check_mark: test(lexer): updated the test for lexer<commit_after>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  _____         _             _
// |_   _|_ _  __| |_ __   ___ | | ___
//   | |/ _` |/ _` | '_ \ / _ \| |/ _ \
//   | | (_| | (_| | |_) | (_) | |  __/
//   |_|\__,_|\__,_| .__/ \___/|_|\___|
//                 |_|
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include "../common/harness.hh"
#include "lexer.hh"

TADPOLE_TEST(TadpoleLexer) {
  using TK = tadpole::TokenKind;
#define TESTEQ(k) TADPOLE_CHECK_EQ(lex.next_token().kind(), k)
#define DUMPLEX() do {\
    for (;;) {\
      auto t = lex.next_token();\
      std::fprintf(stdout, "%-24s | %-32s | %d\n",\
          tadpole::get_kind_name(t.kind()),\
          t.as_cstring(),\
          t.lineno());\
      if (t.kind() == TK::TK_EOF)\
        break;\
    }\
  } while (false)

  {
    std::string s =
      "print(\"Welcome to Tadpole !!!\");\nvar v = 34 * 56 / 22.45 + (88.76 - 27.4);";
    tadpole::Lexer lex(s);
    DUMPLEX();
  }

  {
    std::string s = "var a = 56;";
    tadpole::Lexer lex(s);
    TESTEQ(TK::KW_VAR);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_EQ);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_EOF);
  }

  {
    std::string s = "print(\"23 + 45 =\", 23 + 45);";
    tadpole::Lexer lex(s);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_STRING);
    TESTEQ(TK::TK_COMMA);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_PLUS);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_EOF);
  }

  {
    std::string s = "fn show_msg(msg) { print(msg); }";
    tadpole::Lexer lex(s);
    TESTEQ(TK::KW_FN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_LBRACE);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_RBRACE);
    TESTEQ(TK::TK_EOF);
  }

  {
    std::string s = "fn calc(msg) { print(12 + 45 * 78.23 / 89.11 - 0.67); }";
    tadpole::Lexer lex(s);
    TESTEQ(TK::KW_FN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_LBRACE);
    TESTEQ(TK::TK_IDENTIFIER);
    TESTEQ(TK::TK_LPAREN);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_PLUS);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_STAR);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_SLASH);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_MINUS);
    TESTEQ(TK::TK_NUMERIC);
    TESTEQ(TK::TK_RPAREN);
    TESTEQ(TK::TK_SEMI);
    TESTEQ(TK::TK_RBRACE);
    TESTEQ(TK::TK_EOF);
  }

#undef DUMPLEX
#undef TESTEQ
}
<|endoftext|>


<commit_before>/** \file    extract_normdata_translations.cc
 *  \brief   Extract IxTheo and MACS translations from the normdata file and write it to
 *           language specific text files
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/* The German term is found in Field 150
   Currently there are two different kinds of translations:
   IxTheo-Translations with the following definitions:

   710: Körperschaft - fremdsprachige Äquivalenz
   711: Konferenz - fremdsprachige Äquivalenz
   700: Person - fremdsprachige Äquivalenz
   730: Titel - fremdsprachige Äquivalenz
   750: Sachbegriff - fremdsprachige Äquivalenz
   751: Geografikum - fremdsprachige Äquivalenz

   LoC/Rameau Translations:
   700: Person - Bevorzugter Name in einem anderen Datenbestand
   710: Körperschaft - Bevorzugter Name in einem anderen Datenbestand
   711: Konferenz - Bevorzugte Benennung in einem anderen Datenbestand
   730: Einheitstitel - Bevorzugter Name in einem anderen Datenbestand
   750: Sachbegriff - Bevorzugte Benennung in einem anderen Datenbestand
   751: Geografikum - Bevorzugter Name in einem anderen Datenbestand
*/
#include <iostream>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

// Languages to handle
const unsigned int NUMBER_OF_LANGUAGES = 2;
const std::vector<std::string> languages_to_create{ "en", "fr" };
enum Languages { EN, FR };

void Usage() {
    std::cerr << "Usage: " << ::progname << " norm_data_marc_input extracted_translations\n";
    std::exit(EXIT_FAILURE);
}

void AugmentIxTheoTagWithLanguage(const MarcUtil::Record &record, const std::string &tag, std::vector<std::string> * const translations) {
    auto ixtheo_pos(std::find(translations->begin(), translations->end(), "IxTheo"));
    if (ixtheo_pos != translations->end()) {
        std::vector<std::string> ixtheo_lang_codes;
        record.extractSubfields(tag, "9", &ixtheo_lang_codes);
        bool already_found_ixtheo_translation(false);
        for (const auto &lang_code : ixtheo_lang_codes) {
            if (lang_code[0] != 'L')
                continue;
            if (already_found_ixtheo_translation)
                continue;
            if (lang_code.find("eng") != std::string::npos and *ixtheo_pos != "IxTheo_eng") {
                *ixtheo_pos += + "_eng";
                already_found_ixtheo_translation = true;
            } else if (lang_code.find("fra") != std::string::npos and *ixtheo_pos != "IxTheo_fra") {
                *ixtheo_pos += "_fra";
                already_found_ixtheo_translation = true;
            } else
                Warning("Unsupported language code \"" + lang_code + "\" for PPN " + record.getFields()[0]);
        }
    }
}

void ExtractTranslations(File* const marc_norm_input,
                         const std::string german_term_field_spec,
                         const std::string translation_field_spec,
                         std::map<std::string, std::string> term_to_translation_maps[])
{
    std::set<std::string> german_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(german_term_field_spec, ':', &german_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    std::set<std::string> translation_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(translation_field_spec, ':', &translation_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    if (unlikely(not(german_tags_and_subfield_codes.size() == translation_tags_and_subfield_codes.size())))
        Error("ExtractTranslations: Number of German fields and number of translation fields must be equal");

    unsigned count(0);

    while (const MarcUtil::Record record = MarcUtil::Record::XmlFactory(marc_norm_input)) {
        std::map<std::string, std::vector<std::string>> all_translations;

        for (auto german_and_translations_it(std::make_pair(german_tags_and_subfield_codes.cbegin(), translation_tags_and_subfield_codes.cbegin()));
             german_and_translations_it.first != german_tags_and_subfield_codes.cend();
             ++german_and_translations_it.first, ++german_and_translations_it.second) {

             const std::string german_tag((*german_and_translations_it.first).substr(0, 3));
             const std::string german_subfields((*german_and_translations_it.first).substr(3));
             const std::string translation_tag((*german_and_translations_it.second).substr(0, 3));
             const std::string translation_subfields((*german_and_translations_it.second).substr(3));

             for (auto subfield_iterator = std::make_pair(german_subfields.begin(), translation_subfields.cbegin());
                  subfield_iterator.first != german_subfields.cend();
                  ++subfield_iterator.first, ++subfield_iterator.second) {
                  std::vector<std::string> german_term_for_one_field;
                  record.extractSubfields(german_tag, std::string(1, *subfield_iterator.first), &german_term_for_one_field);

                  std::vector<std::string> translations;
                  // Always extract subfield 2 where "IxTheo" is located
                  record.extractSubfields(translation_tag, std::string(1, *subfield_iterator.second) + "2", &translations);

                  // For IxTheo-Translations add the language code in the same field
                  AugmentIxTheoTagWithLanguage(record, translation_tag, &translations);
                  if (not(german_term_for_one_field.empty()))
                      all_translations.insert(std::make_pair(StringUtil::Join(german_term_for_one_field, ' '), translations));
            }
        }

        for (auto all_translations_it = all_translations.begin(); all_translations_it != all_translations.end(); ++all_translations_it) {
            std::string german_term = all_translations_it->first;

            for (auto translation_vector_it = all_translations_it->second.begin(); translation_vector_it != all_translations_it->second.end(); ++translation_vector_it) {

                if (*translation_vector_it == "IxTheo_eng") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "IxTheo_fra") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "lcsh") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "ram") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                }
           }
       }
    ++count;
    }
}

std::unique_ptr<File> OpenInputFile(const std::string &filename) {
    std::string mode("r");
    if (MediaTypeUtil::GetFileMediaType(filename) == "application/lz4")
        mode += "u";
    std::unique_ptr<File> file(new File(filename, mode));
    if (file->fail())
        Error("can't open \"" + filename + "\" for reading!");

    return file;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string norm_data_marc_input_filename(argv[1]);
    std::unique_ptr<File> norm_data_marc_input(OpenInputFile(norm_data_marc_input_filename));

    const std::string extracted_translations_filename(argv[2]);
    if (unlikely(norm_data_marc_input_filename == extracted_translations_filename))
        Error("Norm data input file name equals output file name!");

    std::string output_mode("w");
    if (norm_data_marc_input->isCompressingOrUncompressing())
        output_mode += 'c';

    // Create a file for each language
    std::vector<std::string> output_file_components;
    if (unlikely(StringUtil::Split(extracted_translations_filename, ".", &output_file_components) < 1))
        Error("extracted_translations_filename " + extracted_translations_filename + " is not valid");

    File *lang_files[NUMBER_OF_LANGUAGES];

    // Derive output components from given input filename
    std::string extension = (output_file_components.size() > 1) ? output_file_components.back() : "";
    std::string basename;
    if (not extension.empty())
        output_file_components.pop_back();
    basename = StringUtil::Join(output_file_components, ".");

    // Assemble output filename
    unsigned i(0);
    for (auto lang : languages_to_create) {
         lang = StringUtil::Trim(lang);

         const std::string lang_file_name_str(extension.empty() ? basename + "_" + lang : basename + "_" + lang + "." + extension);

         lang_files[i] = new File(lang_file_name_str, output_mode);
         if (lang_files[i]->fail())
             Error("can't open \"" + lang_file_name_str + "\" for writing!");
         ++i;
    }

    try {
        std::map<std::string, std::string> term_to_translation_maps[NUMBER_OF_LANGUAGES];

        ExtractTranslations(norm_data_marc_input.get(),
                            "100a:110a:111a:130a:150a:151a",
                            "700a:710a:711a:730a:750a:751a",
                            term_to_translation_maps);
        for (auto line : term_to_translation_maps[EN])
             *(lang_files[EN]) << line.first << '|' << line.second << '\n';

        for (auto line : term_to_translation_maps[FR])
             *(lang_files[FR]) << line.first << '|' << line.second << '\n';

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}

<commit_msg>More minor improvements<commit_after>/** \file    extract_normdata_translations.cc
 *  \brief   Extract IxTheo and MACS translations from the normdata file and write it to
 *           language specific text files
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/* The German term is found in Field 150
   Currently there are two different kinds of translations:
   IxTheo-Translations with the following definitions:

   710: Körperschaft - fremdsprachige Äquivalenz
   711: Konferenz - fremdsprachige Äquivalenz
   700: Person - fremdsprachige Äquivalenz
   730: Titel - fremdsprachige Äquivalenz
   750: Sachbegriff - fremdsprachige Äquivalenz
   751: Geografikum - fremdsprachige Äquivalenz

   LoC/Rameau Translations:
   700: Person - Bevorzugter Name in einem anderen Datenbestand
   710: Körperschaft - Bevorzugter Name in einem anderen Datenbestand
   711: Konferenz - Bevorzugte Benennung in einem anderen Datenbestand
   730: Einheitstitel - Bevorzugter Name in einem anderen Datenbestand
   750: Sachbegriff - Bevorzugte Benennung in einem anderen Datenbestand
   751: Geografikum - Bevorzugter Name in einem anderen Datenbestand
*/
#include <iostream>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

// Languages to handle
const unsigned int NUMBER_OF_LANGUAGES = 2;
const std::vector<std::string> languages_to_create{ "en", "fr" };
enum Languages { EN, FR };

void Usage() {
    std::cerr << "Usage: " << ::progname << " norm_data_marc_input extracted_translations\n";
    std::exit(EXIT_FAILURE);
}

void AugmentIxTheoTagWithLanguage(const MarcUtil::Record &record, const std::string &tag, std::vector<std::string> * const translations) {
    auto ixtheo_pos(std::find(translations->begin(), translations->end(), "IxTheo"));
    if (ixtheo_pos != translations->end()) {
        std::vector<std::string> ixtheo_lang_codes;
        record.extractSubfields(tag, "9", &ixtheo_lang_codes);
        bool already_found_ixtheo_translation(false);
        for (const auto &lang_code : ixtheo_lang_codes) {
            if (lang_code[0] != 'L')
                continue;
            if (already_found_ixtheo_translation)
                continue;
            if (lang_code.find("eng") != std::string::npos and *ixtheo_pos != "IxTheo_eng") {
                *ixtheo_pos += + "_eng";
                already_found_ixtheo_translation = true;
            } else if (lang_code.find("fra") != std::string::npos and *ixtheo_pos != "IxTheo_fra") {
                *ixtheo_pos += "_fra";
                already_found_ixtheo_translation = true;
            } else
                Warning("Unsupported language code \"" + lang_code + "\" for PPN " + record.getFields()[0]);
        }
    }
}

void ExtractTranslations(File * const marc_norm_input,
                         const std::string german_term_field_spec,
                         const std::string translation_field_spec,
                         std::map<std::string, std::string> term_to_translation_maps[])
{
    std::set<std::string> german_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(german_term_field_spec, ':', &german_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    std::set<std::string> translation_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(translation_field_spec, ':', &translation_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    if (unlikely(not(german_tags_and_subfield_codes.size() == translation_tags_and_subfield_codes.size())))
        Error("ExtractTranslations: Number of German fields and number of translation fields must be equal");

    unsigned count(0);

    while (const MarcUtil::Record record = MarcUtil::Record::XmlFactory(marc_norm_input)) {
        std::map<std::string, std::vector<std::string>> all_translations;

        for (auto german_and_translations_it(std::make_pair(german_tags_and_subfield_codes.cbegin(), translation_tags_and_subfield_codes.cbegin()));
             german_and_translations_it.first != german_tags_and_subfield_codes.cend();
             ++german_and_translations_it.first, ++german_and_translations_it.second) {

             const std::string german_tag((*german_and_translations_it.first).substr(0, 3));
             const std::string german_subfields((*german_and_translations_it.first).substr(3));
             const std::string translation_tag((*german_and_translations_it.second).substr(0, 3));
             const std::string translation_subfields((*german_and_translations_it.second).substr(3));

             for (auto subfield_iterator = std::make_pair(german_subfields.begin(), translation_subfields.cbegin());
                  subfield_iterator.first != german_subfields.cend();
                  ++subfield_iterator.first, ++subfield_iterator.second) {
                  std::vector<std::string> german_term_for_one_field;
                  record.extractSubfields(german_tag, std::string(1, *subfield_iterator.first), &german_term_for_one_field);

                  std::vector<std::string> translations;
                  // Always extract subfield 2 where "IxTheo" is located
                  record.extractSubfields(translation_tag, std::string(1, *subfield_iterator.second) + "2", &translations);

                  // For IxTheo-Translations add the language code in the same field
                  AugmentIxTheoTagWithLanguage(record, translation_tag, &translations);
                  if (not german_term_for_one_field.empty())
                      all_translations.insert(std::make_pair(StringUtil::Join(german_term_for_one_field, ' '), translations));
            }
        }

        for (auto all_translations_it = all_translations.begin(); all_translations_it != all_translations.end(); ++all_translations_it) {
            std::string german_term = all_translations_it->first;

            for (auto translation_vector_it = all_translations_it->second.begin(); translation_vector_it != all_translations_it->second.end(); ++translation_vector_it) {

                if (*translation_vector_it == "IxTheo_eng")
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                else if (*translation_vector_it == "IxTheo_fra")
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                else if (*translation_vector_it == "lcsh")
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                else if (*translation_vector_it == "ram")
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
           }
       }
    ++count;
    }
}

std::unique_ptr<File> OpenInputFile(const std::string &filename) {
    std::string mode("r");
    if (MediaTypeUtil::GetFileMediaType(filename) == "application/lz4")
        mode += "u";
    std::unique_ptr<File> file(new File(filename, mode));
    if (file->fail())
        Error("can't open \"" + filename + "\" for reading!");

    return file;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string norm_data_marc_input_filename(argv[1]);
    std::unique_ptr<File> norm_data_marc_input(OpenInputFile(norm_data_marc_input_filename));

    const std::string extracted_translations_filename(argv[2]);
    if (unlikely(norm_data_marc_input_filename == extracted_translations_filename))
        Error("Norm data input file name equals output file name!");

    std::string output_mode("w");
    if (norm_data_marc_input->isCompressingOrUncompressing())
        output_mode += 'c';

    // Create a file for each language
    std::vector<std::string> output_file_components;
    if (unlikely(StringUtil::Split(extracted_translations_filename, ".", &output_file_components) < 1))
        Error("extracted_translations_filename " + extracted_translations_filename + " is not valid");

    File *lang_files[NUMBER_OF_LANGUAGES];

    // Derive output components from given input filename
    std::string extension = (output_file_components.size() > 1) ? output_file_components.back() : "";
    std::string basename;
    if (not extension.empty())
        output_file_components.pop_back();
    basename = StringUtil::Join(output_file_components, ".");

    // Assemble output filename
    unsigned i(0);
    for (auto lang : languages_to_create) {
         lang = StringUtil::Trim(lang);

         const std::string lang_file_name_str(extension.empty() ? basename + "_" + lang : basename + "_" + lang + "." + extension);

         lang_files[i] = new File(lang_file_name_str, output_mode);
         if (lang_files[i]->fail())
             Error("can't open \"" + lang_file_name_str + "\" for writing!");
         ++i;
    }

    try {
        std::map<std::string, std::string> term_to_translation_maps[NUMBER_OF_LANGUAGES];

        ExtractTranslations(norm_data_marc_input.get(),
                            "100a:110a:111a:130a:150a:151a",
                            "700a:710a:711a:730a:750a:751a",
                            term_to_translation_maps);
        for (const auto &line : term_to_translation_maps[EN])
             *(lang_files[EN]) << line.first << '|' << line.second << '\n';

        for (const auto &line : term_to_translation_maps[FR])
             *(lang_files[FR]) << line.first << '|' << line.second << '\n';

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}

<|endoftext|>


<commit_before>#include <stdio.h>

#include "kernellogger.h"
#include "multiboot.h"
#include "paging.h"
#include "system.h"
#include "utils.h"

extern "C"
void pageFault(const registers* regs)
{
    klog.logError(PAGING_TAG, "Page fault!");
    klog.logError(PAGING_TAG, "Error code: {}", regs->errCode);

    char msg[128];
    int idx = 0;

    if ( (regs->errCode & PAGE_ERROR_USER) != 0 )
    {
        idx += sprintf(msg + idx, "A user process ");
    }
    else
    {
        idx += sprintf(msg + idx, "A supervisory process ");
    }

    if ( (regs->errCode & PAGE_ERROR_WRITE) != 0 )
    {
        idx += sprintf(msg + idx, "tried to write to ");
    }
    else
    {
        idx += sprintf(msg + idx, "tried to read from ");
    }

    if ( (regs->errCode & PAGE_ERROR_PRESENT) != 0 )
    {
        idx += sprintf(msg + idx, "a page and caused a protection fault.");
    }
    else
    {
        idx += sprintf(msg + idx, "a non-present page.");
    }

    klog.logError(PAGING_TAG, msg);
    klog.logError(PAGING_TAG, "Address: {0>8}", getRegCR2());

    PANIC("Page fault!");
}

void configPaging()
{
    // register page fault handler
    registerIsrHandler(ISR_PAGE_FAULT, pageFault);
}

void mapPageTable(uint32_t* pageDir, uint32_t pageTableAddr, int pageDirIdx, bool user)
{
    if (pageDirIdx < 0 || pageDirIdx >= 1024)
    {
        PANIC("Invalid page directory index.");
    }

    // add the page table to the page directory
    uint32_t pageDirEntry = 0;
    pageDirEntry |= pageTableAddr & PAGE_DIR_ADDRESS;       // add new address
    pageDirEntry |= PAGE_DIR_READ_WRITE | PAGE_DIR_PRESENT; // set read/write and present bits
    if (user)
    {
        pageDirEntry |= PAGE_DIR_USER; // set user privilege
    }
    pageDir[pageDirIdx] = pageDirEntry;
}

void mapPage(uint32_t* pageTable, uint32_t virtualAddr, uint32_t physicalAddr, bool user)
{
    // calculate the page table index
    int pageTableIdx = (virtualAddr >> 12) & PAGE_TABLE_INDEX_MASK;

    // get the page entry from the page table
    uint32_t pageTableEntry = pageTable[pageTableIdx];

    // add the page address to the page table
    pageTableEntry &= ~PAGE_TABLE_ADDRESS;                        // clear previous address
    pageTableEntry |= physicalAddr & PAGE_TABLE_ADDRESS;          // add new address
    pageTableEntry |= PAGE_TABLE_READ_WRITE | PAGE_TABLE_PRESENT; // set read/write and present bits
    if (user)
    {
        pageTableEntry |= PAGE_TABLE_USER; // set user privilege
    }
    pageTable[pageTableIdx] = pageTableEntry;
}

bool mapPage(int pageDirIdx, uint32_t* pageTable, uint32_t& virtualAddr, uint32_t physicalAddr, bool user)
{
    for (int idx = 0; idx < PAGE_TABLE_NUM_ENTRIES; ++idx)
    {
        uint32_t entry = pageTable[idx];
        if ( (entry & PAGE_TABLE_PRESENT) == 0 )
        {
            uint32_t newEntry = physicalAddr & PAGE_TABLE_ADDRESS;  // set address
            newEntry |= PAGE_TABLE_READ_WRITE | PAGE_TABLE_PRESENT; // set read/write and present bits
            if (user)
            {
                newEntry |= PAGE_TABLE_USER; // set user privilege
            }

            pageTable[idx] = newEntry;

            virtualAddr = (pageDirIdx << 22) | (idx << 12);
            return true;
        }
    }

    return false;
}

void unmapPage(uint32_t* pageTable, uint32_t virtualAddr)
{
    // calculate the page table index
    int pageTableIdx = (virtualAddr >> 12) & PAGE_TABLE_INDEX_MASK;

    // clear the page table entry
    pageTable[pageTableIdx] = 0;

    // invalidate page in TLB
    invalidatePage(virtualAddr);
}

void mapModules(const multiboot_info* mbootInfo)
{
    uint32_t modAddr = mbootInfo->mods_addr + KERNEL_VIRTUAL_BASE;
    for (uint32_t i = 0; i < mbootInfo->mods_count; ++i)
    {
        const multiboot_mod_list* module = reinterpret_cast<const multiboot_mod_list*>(modAddr);

        uint32_t startPageAddr = align(module->mod_start, PAGE_SIZE, false);

        // end page is the page after the last page the module is in
        uint32_t endPageAddr = align(module->mod_end, PAGE_SIZE);

        for (uint32_t pageAddr = startPageAddr; pageAddr < endPageAddr; pageAddr += PAGE_SIZE)
        {
            uint32_t virtualAddr = pageAddr + KERNEL_VIRTUAL_BASE;
            mapPage(getKernelPageTableStart(), virtualAddr, pageAddr);
        }

        modAddr += sizeof(multiboot_mod_list);
    }
}
<commit_msg>Bugfix: Page fault error message did not print address in hex format<commit_after>#include <stdio.h>

#include "kernellogger.h"
#include "multiboot.h"
#include "paging.h"
#include "system.h"
#include "utils.h"

extern "C"
void pageFault(const registers* regs)
{
    klog.logError(PAGING_TAG, "Page fault!");
    klog.logError(PAGING_TAG, "Error code: {}", regs->errCode);

    char msg[128];
    int idx = 0;

    if ( (regs->errCode & PAGE_ERROR_USER) != 0 )
    {
        idx += sprintf(msg + idx, "A user process ");
    }
    else
    {
        idx += sprintf(msg + idx, "A supervisory process ");
    }

    if ( (regs->errCode & PAGE_ERROR_WRITE) != 0 )
    {
        idx += sprintf(msg + idx, "tried to write to ");
    }
    else
    {
        idx += sprintf(msg + idx, "tried to read from ");
    }

    if ( (regs->errCode & PAGE_ERROR_PRESENT) != 0 )
    {
        idx += sprintf(msg + idx, "a page and caused a protection fault.");
    }
    else
    {
        idx += sprintf(msg + idx, "a non-present page.");
    }

    klog.logError(PAGING_TAG, msg);
    klog.logError(PAGING_TAG, "Address: {x0>8}", getRegCR2());

    PANIC("Page fault!");
}

void configPaging()
{
    // register page fault handler
    registerIsrHandler(ISR_PAGE_FAULT, pageFault);
}

void mapPageTable(uint32_t* pageDir, uint32_t pageTableAddr, int pageDirIdx, bool user)
{
    if (pageDirIdx < 0 || pageDirIdx >= 1024)
    {
        PANIC("Invalid page directory index.");
    }

    // add the page table to the page directory
    uint32_t pageDirEntry = 0;
    pageDirEntry |= pageTableAddr & PAGE_DIR_ADDRESS;       // add new address
    pageDirEntry |= PAGE_DIR_READ_WRITE | PAGE_DIR_PRESENT; // set read/write and present bits
    if (user)
    {
        pageDirEntry |= PAGE_DIR_USER; // set user privilege
    }
    pageDir[pageDirIdx] = pageDirEntry;
}

void mapPage(uint32_t* pageTable, uint32_t virtualAddr, uint32_t physicalAddr, bool user)
{
    // calculate the page table index
    int pageTableIdx = (virtualAddr >> 12) & PAGE_TABLE_INDEX_MASK;

    // get the page entry from the page table
    uint32_t pageTableEntry = pageTable[pageTableIdx];

    // add the page address to the page table
    pageTableEntry &= ~PAGE_TABLE_ADDRESS;                        // clear previous address
    pageTableEntry |= physicalAddr & PAGE_TABLE_ADDRESS;          // add new address
    pageTableEntry |= PAGE_TABLE_READ_WRITE | PAGE_TABLE_PRESENT; // set read/write and present bits
    if (user)
    {
        pageTableEntry |= PAGE_TABLE_USER; // set user privilege
    }
    pageTable[pageTableIdx] = pageTableEntry;
}

bool mapPage(int pageDirIdx, uint32_t* pageTable, uint32_t& virtualAddr, uint32_t physicalAddr, bool user)
{
    for (int idx = 0; idx < PAGE_TABLE_NUM_ENTRIES; ++idx)
    {
        uint32_t entry = pageTable[idx];
        if ( (entry & PAGE_TABLE_PRESENT) == 0 )
        {
            uint32_t newEntry = physicalAddr & PAGE_TABLE_ADDRESS;  // set address
            newEntry |= PAGE_TABLE_READ_WRITE | PAGE_TABLE_PRESENT; // set read/write and present bits
            if (user)
            {
                newEntry |= PAGE_TABLE_USER; // set user privilege
            }

            pageTable[idx] = newEntry;

            virtualAddr = (pageDirIdx << 22) | (idx << 12);
            return true;
        }
    }

    return false;
}

void unmapPage(uint32_t* pageTable, uint32_t virtualAddr)
{
    // calculate the page table index
    int pageTableIdx = (virtualAddr >> 12) & PAGE_TABLE_INDEX_MASK;

    // clear the page table entry
    pageTable[pageTableIdx] = 0;

    // invalidate page in TLB
    invalidatePage(virtualAddr);
}

void mapModules(const multiboot_info* mbootInfo)
{
    uint32_t modAddr = mbootInfo->mods_addr + KERNEL_VIRTUAL_BASE;
    for (uint32_t i = 0; i < mbootInfo->mods_count; ++i)
    {
        const multiboot_mod_list* module = reinterpret_cast<const multiboot_mod_list*>(modAddr);

        uint32_t startPageAddr = align(module->mod_start, PAGE_SIZE, false);

        // end page is the page after the last page the module is in
        uint32_t endPageAddr = align(module->mod_end, PAGE_SIZE);

        for (uint32_t pageAddr = startPageAddr; pageAddr < endPageAddr; pageAddr += PAGE_SIZE)
        {
            uint32_t virtualAddr = pageAddr + KERNEL_VIRTUAL_BASE;
            mapPage(getKernelPageTableStart(), virtualAddr, pageAddr);
        }

        modAddr += sizeof(multiboot_mod_list);
    }
}
<|endoftext|>


<commit_before>/** \file    extract_normdata_translations.cc
 *  \brief   Extract IxTheo and MACS translations from the normdata file and write it to
 *           language specific text files
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/* The German term is found in Field 150
   Currently there are two different kinds of translations:
   IxTheo-Translations with the following definitions:

   710: Körperschaft - fremdsprachige Äquivalenz
   711: Konferenz - fremdsprachige Äquivalenz
   700: Person - fremdsprachige Äquivalenz
   730: Titel - fremdsprachige Äquivalenz
   750: Sachbegriff - fremdsprachige Äquivalenz
   751: Geografikum - fremdsprachige Äquivalenz

   LoC/Rameau Translations:
   700: Person - Bevorzugter Name in einem anderen Datenbestand
   710: Körperschaft - Bevorzugter Name in einem anderen Datenbestand
   711: Konferenz - Bevorzugte Benennung in einem anderen Datenbestand
   730: Einheitstitel - Bevorzugter Name in einem anderen Datenbestand
   750: Sachbegriff - Bevorzugte Benennung in einem anderen Datenbestand
   751: Geografikum - Bevorzugter Name in einem anderen Datenbestand
*/
#include <iostream>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

// Languages to handle
const unsigned int NUMBER_OF_LANGUAGES = 2;
const std::vector<std::string> languages_to_create{ "en", "fr" };
enum Languages { EN, FR };

void Usage() {
    std::cerr << "Usage: " << ::progname << " norm_data_marc_input extracted_translations\n";
    std::exit(EXIT_FAILURE);
}

void AugmentIxTheoTagWithLanguage(const MarcUtil::Record &record, const std::string &tag, std::vector<std::string> * const translations) {
    auto ixtheo_pos(std::find(translations->begin(), translations->end(), "IxTheo"));
    if (ixtheo_pos != translations->end()) {
        std::vector<std::string> ixtheo_lang_codes;
        record.extractSubfields(tag, "9", &ixtheo_lang_codes);
        bool already_found_ixtheo_translation(false);
        for (const auto &lang_code : ixtheo_lang_codes) {
            if (lang_code[0] != 'L')
                continue;
            if (already_found_ixtheo_translation)
                continue;
            if (lang_code.find("eng") != std::string::npos and *ixtheo_pos != "IxTheo_eng") {
                *ixtheo_pos += + "_eng";
                already_found_ixtheo_translation = true;
            } else if (lang_code.find("fra") != std::string::npos and *ixtheo_pos != "IxTheo_fra") {
                *ixtheo_pos += "_fra";
                already_found_ixtheo_translation = true;
            } else
                Warning("Unsupported language code \"" + lang_code + "\" for PPN " + record.getFields()[0]);
        }
    }
}

void ExtractTranslations(File* const marc_norm_input,
                         const std::string german_term_field_spec,
                         const std::string translation_field_spec,
                         std::map<std::string, std::string> term_to_translation_maps[])
{
    std::set<std::string> german_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(german_term_field_spec, ':', &german_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    std::set<std::string> translation_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(translation_field_spec, ':', &translation_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    if (unlikely(not(german_tags_and_subfield_codes.size() == translation_tags_and_subfield_codes.size())))
        Error("ExtractTranslations: Number of German fields and number of translation fields must be equal");

    unsigned count(0);

    while (const MarcUtil::Record record = MarcUtil::Record::XmlFactory(marc_norm_input)) {
        std::map<std::string, std::vector<std::string>> all_translations;

        for (auto german_and_translations_it(std::make_pair(german_tags_and_subfield_codes.cbegin(), translation_tags_and_subfield_codes.cbegin()));
             german_and_translations_it.first != german_tags_and_subfield_codes.cend();
             ++german_and_translations_it.first, ++german_and_translations_it.second) {

             const std::string german_tag((*german_and_translations_it.first).substr(0, 3));
             const std::string german_subfields((*german_and_translations_it.first).substr(3));
             const std::string translation_tag((*german_and_translations_it.second).substr(0, 3));
             const std::string translation_subfields((*german_and_translations_it.second).substr(3));

             for (auto subfield_iterator = std::make_pair(german_subfields.begin(), translation_subfields.cbegin());
                  subfield_iterator.first != german_subfields.cend();
                  ++subfield_iterator.first, ++subfield_iterator.second) {
                  std::vector<std::string> german_term_for_one_field;
                  record.extractSubfields(german_tag, std::string(1, *subfield_iterator.first), &german_term_for_one_field);

                  std::vector<std::string> translations;
                  // Always extract subfield 2 where "IxTheo" is located
                  record.extractSubfields(translation_tag, std::string(1, *subfield_iterator.second) + "2", &translations);

                  // For IxTheo-Translations add the language code in the same field
                  AugmentIxTheoTagWithLanguage(record, translation_tag, &translations);
                  if (not(german_term_for_one_field.empty()))
                      all_translations.insert(std::make_pair(StringUtil::Join(german_term_for_one_field, ' '), translations));
            }
        }

        for (auto all_translations_it = all_translations.begin(); all_translations_it != all_translations.end(); ++all_translations_it) {
            std::string german_term = all_translations_it->first;

            for (auto translation_vector_it = all_translations_it->second.begin(); translation_vector_it != all_translations_it->second.end(); ++translation_vector_it) {

                if (*translation_vector_it == "IxTheo_eng") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "IxTheo_fra") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "lcsh") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "ram") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                }
           }
       }
    ++count;
    }
}

std::unique_ptr<File> OpenInputFile(const std::string &filename) {
    std::string mode("r");
    if (MediaTypeUtil::GetFileMediaType(filename) == "application/lz4")
        mode += "u";
    std::unique_ptr<File> file(new File(filename, mode));
    if (file->fail())
        Error("can't open \"" + filename + "\" for reading!");

    return file;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string norm_data_marc_input_filename(argv[1]);
    std::unique_ptr<File> norm_data_marc_input(OpenInputFile(norm_data_marc_input_filename));

    const std::string extracted_translations_filename(argv[2]);
    if (unlikely(norm_data_marc_input_filename == extracted_translations_filename))
        Error("Norm data input file name equals output file name!");

    std::string output_mode("w");
    if (norm_data_marc_input->isCompressingOrUncompressing())
        output_mode += 'c';

    // Create a file for each language
    std::vector<std::string> output_file_components;
    if (unlikely(StringUtil::Split(extracted_translations_filename, ".", &output_file_components) < 1))
        Error("extracted_translations_filename " + extracted_translations_filename + " is not valid");

    File *lang_files[NUMBER_OF_LANGUAGES];
    unsigned i(0);

    // Derive output components from given input filename
    std::string extension = (output_file_components.size() > 1) ? output_file_components.back() : "";
    std::string basename;
    if (not extension.empty())
        output_file_components.pop_back();
    basename = StringUtil::Join(output_file_components, ".");

    // Assemble output filename
    for (auto lang : languages_to_create) {
         lang = StringUtil::Trim(lang);

         std::string lang_file_name_str =
             (extension != "") ? basename + "_" + lang + "." + extension : basename + "_" + lang;

         lang_files[i] = new File(lang_file_name_str, output_mode);
         if (lang_files[i]->fail())
             Error("can't open \"" + lang_file_name_str + "\" for writing!");
         ++i;
    }

    try {
        std::map<std::string, std::string> term_to_translation_maps[NUMBER_OF_LANGUAGES];

        ExtractTranslations(norm_data_marc_input.get(),
                            "100a:110a:111a:130a:150a:151a",
                            "700a:710a:711a:730a:750a:751a",
                            term_to_translation_maps);
        for (auto line : term_to_translation_maps[EN])
             *(lang_files[EN]) << line.first << '|' << line.second << '\n';

        for (auto line : term_to_translation_maps[FR])
             *(lang_files[FR]) << line.first << '|' << line.second << '\n';

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}

<commit_msg>Minor improvements<commit_after>/** \file    extract_normdata_translations.cc
 *  \brief   Extract IxTheo and MACS translations from the normdata file and write it to
 *           language specific text files
 *  \author  Johannes Riedl
 */

/*
    Copyright (C) 2016, Library of the University of Tübingen

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/* The German term is found in Field 150
   Currently there are two different kinds of translations:
   IxTheo-Translations with the following definitions:

   710: Körperschaft - fremdsprachige Äquivalenz
   711: Konferenz - fremdsprachige Äquivalenz
   700: Person - fremdsprachige Äquivalenz
   730: Titel - fremdsprachige Äquivalenz
   750: Sachbegriff - fremdsprachige Äquivalenz
   751: Geografikum - fremdsprachige Äquivalenz

   LoC/Rameau Translations:
   700: Person - Bevorzugter Name in einem anderen Datenbestand
   710: Körperschaft - Bevorzugter Name in einem anderen Datenbestand
   711: Konferenz - Bevorzugte Benennung in einem anderen Datenbestand
   730: Einheitstitel - Bevorzugter Name in einem anderen Datenbestand
   750: Sachbegriff - Bevorzugte Benennung in einem anderen Datenbestand
   751: Geografikum - Bevorzugter Name in einem anderen Datenbestand
*/
#include <iostream>
#include <vector>
#include <cstdlib>
#include "Compiler.h"
#include "MarcUtil.h"
#include "MediaTypeUtil.h"
#include "StringUtil.h"
#include "Subfields.h"
#include "util.h"

// Languages to handle
const unsigned int NUMBER_OF_LANGUAGES = 2;
const std::vector<std::string> languages_to_create{ "en", "fr" };
enum Languages { EN, FR };

void Usage() {
    std::cerr << "Usage: " << ::progname << " norm_data_marc_input extracted_translations\n";
    std::exit(EXIT_FAILURE);
}

void AugmentIxTheoTagWithLanguage(const MarcUtil::Record &record, const std::string &tag, std::vector<std::string> * const translations) {
    auto ixtheo_pos(std::find(translations->begin(), translations->end(), "IxTheo"));
    if (ixtheo_pos != translations->end()) {
        std::vector<std::string> ixtheo_lang_codes;
        record.extractSubfields(tag, "9", &ixtheo_lang_codes);
        bool already_found_ixtheo_translation(false);
        for (const auto &lang_code : ixtheo_lang_codes) {
            if (lang_code[0] != 'L')
                continue;
            if (already_found_ixtheo_translation)
                continue;
            if (lang_code.find("eng") != std::string::npos and *ixtheo_pos != "IxTheo_eng") {
                *ixtheo_pos += + "_eng";
                already_found_ixtheo_translation = true;
            } else if (lang_code.find("fra") != std::string::npos and *ixtheo_pos != "IxTheo_fra") {
                *ixtheo_pos += "_fra";
                already_found_ixtheo_translation = true;
            } else
                Warning("Unsupported language code \"" + lang_code + "\" for PPN " + record.getFields()[0]);
        }
    }
}

void ExtractTranslations(File* const marc_norm_input,
                         const std::string german_term_field_spec,
                         const std::string translation_field_spec,
                         std::map<std::string, std::string> term_to_translation_maps[])
{
    std::set<std::string> german_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(german_term_field_spec, ':', &german_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    std::set<std::string> translation_tags_and_subfield_codes;
    if (unlikely(StringUtil::Split(translation_field_spec, ':', &translation_tags_and_subfield_codes) < 1))
        Error("ExtractTranslations: Need at least one translation field");

    if (unlikely(not(german_tags_and_subfield_codes.size() == translation_tags_and_subfield_codes.size())))
        Error("ExtractTranslations: Number of German fields and number of translation fields must be equal");

    unsigned count(0);

    while (const MarcUtil::Record record = MarcUtil::Record::XmlFactory(marc_norm_input)) {
        std::map<std::string, std::vector<std::string>> all_translations;

        for (auto german_and_translations_it(std::make_pair(german_tags_and_subfield_codes.cbegin(), translation_tags_and_subfield_codes.cbegin()));
             german_and_translations_it.first != german_tags_and_subfield_codes.cend();
             ++german_and_translations_it.first, ++german_and_translations_it.second) {

             const std::string german_tag((*german_and_translations_it.first).substr(0, 3));
             const std::string german_subfields((*german_and_translations_it.first).substr(3));
             const std::string translation_tag((*german_and_translations_it.second).substr(0, 3));
             const std::string translation_subfields((*german_and_translations_it.second).substr(3));

             for (auto subfield_iterator = std::make_pair(german_subfields.begin(), translation_subfields.cbegin());
                  subfield_iterator.first != german_subfields.cend();
                  ++subfield_iterator.first, ++subfield_iterator.second) {
                  std::vector<std::string> german_term_for_one_field;
                  record.extractSubfields(german_tag, std::string(1, *subfield_iterator.first), &german_term_for_one_field);

                  std::vector<std::string> translations;
                  // Always extract subfield 2 where "IxTheo" is located
                  record.extractSubfields(translation_tag, std::string(1, *subfield_iterator.second) + "2", &translations);

                  // For IxTheo-Translations add the language code in the same field
                  AugmentIxTheoTagWithLanguage(record, translation_tag, &translations);
                  if (not(german_term_for_one_field.empty()))
                      all_translations.insert(std::make_pair(StringUtil::Join(german_term_for_one_field, ' '), translations));
            }
        }

        for (auto all_translations_it = all_translations.begin(); all_translations_it != all_translations.end(); ++all_translations_it) {
            std::string german_term = all_translations_it->first;

            for (auto translation_vector_it = all_translations_it->second.begin(); translation_vector_it != all_translations_it->second.end(); ++translation_vector_it) {

                if (*translation_vector_it == "IxTheo_eng") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "IxTheo_fra") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "lcsh") {
                    term_to_translation_maps[EN].emplace(german_term, *(++translation_vector_it));
                } else if (*translation_vector_it == "ram") {
                    term_to_translation_maps[FR].emplace(german_term, *(++translation_vector_it));
                }
           }
       }
    ++count;
    }
}

std::unique_ptr<File> OpenInputFile(const std::string &filename) {
    std::string mode("r");
    if (MediaTypeUtil::GetFileMediaType(filename) == "application/lz4")
        mode += "u";
    std::unique_ptr<File> file(new File(filename, mode));
    if (file->fail())
        Error("can't open \"" + filename + "\" for reading!");

    return file;
}

int main(int argc, char **argv) {
    ::progname = argv[0];

    if (argc != 3)
        Usage();

    const std::string norm_data_marc_input_filename(argv[1]);
    std::unique_ptr<File> norm_data_marc_input(OpenInputFile(norm_data_marc_input_filename));

    const std::string extracted_translations_filename(argv[2]);
    if (unlikely(norm_data_marc_input_filename == extracted_translations_filename))
        Error("Norm data input file name equals output file name!");

    std::string output_mode("w");
    if (norm_data_marc_input->isCompressingOrUncompressing())
        output_mode += 'c';

    // Create a file for each language
    std::vector<std::string> output_file_components;
    if (unlikely(StringUtil::Split(extracted_translations_filename, ".", &output_file_components) < 1))
        Error("extracted_translations_filename " + extracted_translations_filename + " is not valid");

    File *lang_files[NUMBER_OF_LANGUAGES];

    // Derive output components from given input filename
    std::string extension = (output_file_components.size() > 1) ? output_file_components.back() : "";
    std::string basename;
    if (not extension.empty())
        output_file_components.pop_back();
    basename = StringUtil::Join(output_file_components, ".");

    // Assemble output filename
    unsigned i(0);
    for (auto lang : languages_to_create) {
         lang = StringUtil::Trim(lang);

         const std::string lang_file_name_str(extension.empty() ? basename + "_" + lang : basename + "_" + lang + "." + extension);

         lang_files[i] = new File(lang_file_name_str, output_mode);
         if (lang_files[i]->fail())
             Error("can't open \"" + lang_file_name_str + "\" for writing!");
         ++i;
    }

    try {
        std::map<std::string, std::string> term_to_translation_maps[NUMBER_OF_LANGUAGES];

        ExtractTranslations(norm_data_marc_input.get(),
                            "100a:110a:111a:130a:150a:151a",
                            "700a:710a:711a:730a:750a:751a",
                            term_to_translation_maps);
        for (auto line : term_to_translation_maps[EN])
             *(lang_files[EN]) << line.first << '|' << line.second << '\n';

        for (auto line : term_to_translation_maps[FR])
             *(lang_files[FR]) << line.first << '|' << line.second << '\n';

    } catch (const std::exception &x) {
        Error("caught exception: " + std::string(x.what()));
    }
}

<|endoftext|>