<commit_before>/**************************************************************************
 *
 * Copyright 2010 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 **************************************************************************/

#ifndef _TRACE_PARSER_HPP_
#define _TRACE_PARSER_HPP_

#include <cassert>

#include <iostream>
#include <map>
#include <list>
#include <string>

#include <zlib.h>

#include "trace_format.hpp"
#include "trace_model.hpp"

#define TRACE_VERBOSE 0

namespace Trace {

class Parser
{
protected:
   gzFile file;

   typedef std::list<Call *> CallList;
   CallList calls;

   typedef std::map<size_t, Call::Signature *> FunctionMap;
   FunctionMap functions;

   typedef std::map<size_t, Struct::Signature *> StructMap;
   StructMap structs;

   typedef std::map<size_t, Enum *> EnumMap;
   EnumMap enums;

   typedef std::map<size_t, Bitmask::Signature *> BitmaskMap;
   BitmaskMap bitmasks;

   unsigned next_call_no;

public:
   Parser() {
      file = NULL;
      next_call_no = 0;
   }

   ~Parser() {
      close();
   }

   bool open(const char *filename) {
      unsigned long long version;

      file = gzopen(filename, "rb");
      if (!file) {
         return false;
      }

      version = read_uint();
      if (version != TRACE_VERSION) {
         std::cerr << "error: unsupported format version" << version << "\n";
         return false;
      }

      return true;
   }

   void close(void) {
      if (file) {
         gzclose(file);
         file = NULL;
      }
   }

   Call *parse_call(void) {
      do {
         int c = read_byte();
         switch(c) {
         case Trace::EVENT_ENTER:
            parse_enter();
            break;
         case Trace::EVENT_LEAVE:
            return parse_leave();
         case Trace::EVENT_MESSAGE:
            std::cerr << "message: " << read_string() << "\n";
            break;
         default:
            std::cerr << "error: unknown call detail " << c << "\n";
            assert(0);
            /* fallthrough */
         case -1:
            return NULL;
         }
      } while(true);
   }

   void parse_enter(void) {
      size_t id = read_uint();

      Call::Signature *sig;
      FunctionMap::const_iterator it = functions.find(id);
      if (it == functions.end()) {
          sig = new Call::Signature;
          sig->name = read_string();
          unsigned size = read_uint();
          for (unsigned i = 0; i < size; ++i) {
              sig->arg_names.push_back(read_string());
          }
          functions[id] = sig;
      } else {
          sig = it->second;
      }
      assert(sig);

      Call *call = new Call(sig);
      call->no = next_call_no++;

      parse_call_details(call);

      calls.push_back(call);
   }

   Call *parse_leave(void) {
      unsigned call_no = read_uint();

      Call *call = NULL;
      for (CallList::iterator it = calls.begin(); it != calls.end(); ++it) {
         if ((*it)->no == call_no) {
            call = *it;
            calls.erase(it);
            break;
         }
      }
      assert(call);
      if (!call) {
          return NULL;
      }
      parse_call_details(call);
      return call;
   }

   void parse_call_details(Call *call) {
      do {
         int c = read_byte();
         switch(c) {
         case Trace::CALL_END:
            return;
         case Trace::CALL_ARG:
            parse_arg(call);
            break;
         case Trace::CALL_RET:
            call->ret = parse_value();
            break;
         default:
            std::cerr << "error: unknown call detail " << c << "\n";
            assert(0);
            /* fallthrough */
         case -1:
            return;
         }
      } while(true);
   }

   void parse_arg(Call *call) {
      unsigned index = read_uint();
      Value *value = parse_value();
      if (index >= call->args.size()) {
          call->args.resize(index + 1);
      }
      call->args[index] = value;
   }

   Value *parse_value(void) {
      int c;
      c = read_byte();
      switch(c) {
      case Trace::TYPE_NULL:
         return new Null;
      case Trace::TYPE_FALSE:
         return new Bool(false);
      case Trace::TYPE_TRUE:
         return new Bool(true);
      case Trace::TYPE_SINT:
         return parse_sint();
      case Trace::TYPE_UINT:
         return parse_uint();
      case Trace::TYPE_FLOAT:
         return parse_float();
      case Trace::TYPE_DOUBLE:
         return parse_double();
      case Trace::TYPE_STRING:
         return parse_string();
      case Trace::TYPE_ENUM:
         return parse_enum();
      case Trace::TYPE_BITMASK:
         return parse_bitmask();
      case Trace::TYPE_ARRAY:
         return parse_array();
      case Trace::TYPE_STRUCT:
         return parse_struct();
      case Trace::TYPE_BLOB:
         return parse_blob();
      case Trace::TYPE_OPAQUE:
         return parse_opaque();
      default:
         std::cerr << "error: unknown type " << c << "\n";
         assert(0);
         return NULL;
      }
   }

   Value *parse_sint() {
      return new SInt(-(signed long long)read_uint());
   }

   Value *parse_uint() {
      return new UInt(read_uint());
   }

   Value *parse_float() {
      float value;
      gzread(file, &value, sizeof value);
      return new Float(value);
   }

   Value *parse_double() {
      double value;
      gzread(file, &value, sizeof value);
      return new Float(value);
   }

   Value *parse_string() {
      return new String(read_string());
   }

   Value *parse_enum() {
      size_t id = read_uint();
      Enum *sig;
      EnumMap::const_iterator it = enums.find(id);
      if (it == enums.end()) {
          std::string name = read_string();
          Value *value = parse_value();
          sig = new Enum(name, value);
          enums[id] = sig;
      } else {
          sig = it->second;
      }
      assert(sig);
      return sig;
   }

   Value *parse_bitmask() {
      size_t id = read_uint();
      Bitmask::Signature *sig;
      BitmaskMap::const_iterator it = bitmasks.find(id);
      if (it == bitmasks.end()) {
          size_t size = read_uint();
          sig = new Bitmask::Signature(size);
          for (Bitmask::Signature::iterator it = sig->begin(); it != sig->end(); ++it) {
              it->first = read_string();
              it->second = read_uint();
              assert(it->second);
          }
          bitmasks[id] = sig;
      } else {
          sig = it->second;
      }
      assert(sig);

      unsigned long long value = read_uint();

      return new Bitmask(sig, value);
   }

   Value *parse_array(void) {
      size_t len = read_uint();
      Array *array = new Array(len);
      for (size_t i = 0; i < len; ++i) {
         array->values[i] = parse_value();
      }
      return array;
   }

   Value *parse_blob(void) {
      size_t size = read_uint();
      Blob *blob = new Blob(size);
      if (size) {
          gzread(file, blob->buf, size);
      }
      return blob;
   }

   Value *parse_struct() {
      size_t id = read_uint();

      Struct::Signature *sig;
      StructMap::const_iterator it = structs.find(id);
      if (it == structs.end()) {
          sig = new Struct::Signature;
          sig->name = read_string();
          unsigned size = read_uint();
          for (unsigned i = 0; i < size; ++i) {
              sig->member_names.push_back(read_string());
          }
          structs[id] = sig;
      } else {
          sig = it->second;
      }
      assert(sig);

      Struct *value = new Struct(sig);

      for (size_t i = 0; i < sig->member_names.size(); ++i) {
          value->members[i] = parse_value();
      }

      return value;
   }

   Value *parse_opaque() {
      unsigned long long addr;
      addr = read_uint();
      /* XXX */
      return new UInt(addr);
   }

   std::string read_string(void) {
      size_t len = read_uint();
      if (!len) {
         return std::string();
      }
      char * buf = new char[len];
      gzread(file, buf, len);
      std::string value(buf, len);
      delete [] buf;
#if TRACE_VERBOSE
      std::cerr << "\tSTRING \"" << value << "\"\n";
#endif
      return value;
   }

   unsigned long long read_uint(void) {
      unsigned long long value = 0;
      int c;
      unsigned shift = 0;
      do {
         c = gzgetc(file);
         if (c == -1) {
            break;
         }
         value |= (unsigned long long)(c & 0x7f) << shift;
         shift += 7;
      } while(c & 0x80);
#if TRACE_VERBOSE
      std::cerr << "\tUINT " << value << "\n";
#endif
      return value;
   }

   int read_byte(void) {
      int c = gzgetc(file);
#if TRACE_VERBOSE
      if (c < 0)
         std::cerr << "\tEOF" << "\n";
      else
         std::cerr << "\tBYTE 0x" << std::hex << c << std::dec << "\n";
#endif
      return c;
   }
};

} /* namespace Trace */

#endif /* _TRACE_PARSER_HPP_ */
<commit_msg>Use vectors instead of maps.<commit_after>/**************************************************************************
 *
 * Copyright 2010 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 **************************************************************************/

#ifndef _TRACE_PARSER_HPP_
#define _TRACE_PARSER_HPP_

#include <cassert>

#include <iostream>
#include <map>
#include <list>
#include <string>

#include <zlib.h>

#include "trace_format.hpp"
#include "trace_model.hpp"

#define TRACE_VERBOSE 0

namespace Trace {

class Parser
{
protected:
   gzFile file;

   typedef std::list<Call *> CallList;
   CallList calls;

   typedef std::vector<Call::Signature *> FunctionMap;
   FunctionMap functions;

   typedef std::vector<Struct::Signature *> StructMap;
   StructMap structs;

   typedef std::vector<Enum *> EnumMap;
   EnumMap enums;

   typedef std::vector<Bitmask::Signature *> BitmaskMap;
   BitmaskMap bitmasks;

   unsigned next_call_no;

public:
   Parser() {
      file = NULL;
      next_call_no = 0;
   }

   ~Parser() {
      close();
   }

   bool open(const char *filename) {
      unsigned long long version;

      file = gzopen(filename, "rb");
      if (!file) {
         return false;
      }

      version = read_uint();
      if (version != TRACE_VERSION) {
         std::cerr << "error: unsupported format version" << version << "\n";
         return false;
      }

      return true;
   }

   void close(void) {
      if (file) {
         gzclose(file);
         file = NULL;
      }
   }

   Call *parse_call(void) {
      do {
         int c = read_byte();
         switch(c) {
         case Trace::EVENT_ENTER:
            parse_enter();
            break;
         case Trace::EVENT_LEAVE:
            return parse_leave();
         case Trace::EVENT_MESSAGE:
            std::cerr << "message: " << read_string() << "\n";
            break;
         default:
            std::cerr << "error: unknown call detail " << c << "\n";
            assert(0);
            /* fallthrough */
         case -1:
            return NULL;
         }
      } while(true);
   }

   /**
    * Helper function to lookup an ID in a vector, resizing the vector if it doesn't fit.
    */
   template<class T>
   T *lookup(std::vector<T *> &map, size_t index) {
      if (index >= map.size()) {
         map.resize(index + 1);
         return NULL;
      } else {
         return map[index];
      }
   }

   void parse_enter(void) {
      size_t id = read_uint();

      Call::Signature *sig = lookup(functions, id);
      if (!sig) {
          sig = new Call::Signature;
          sig->name = read_string();
          unsigned size = read_uint();
          for (unsigned i = 0; i < size; ++i) {
              sig->arg_names.push_back(read_string());
          }
          functions[id] = sig;
      }
      assert(sig);

      Call *call = new Call(sig);
      call->no = next_call_no++;

      parse_call_details(call);

      calls.push_back(call);
   }

   Call *parse_leave(void) {
      unsigned call_no = read_uint();

      Call *call = NULL;
      for (CallList::iterator it = calls.begin(); it != calls.end(); ++it) {
         if ((*it)->no == call_no) {
            call = *it;
            calls.erase(it);
            break;
         }
      }
      assert(call);
      if (!call) {
          return NULL;
      }
      parse_call_details(call);
      return call;
   }

   void parse_call_details(Call *call) {
      do {
         int c = read_byte();
         switch(c) {
         case Trace::CALL_END:
            return;
         case Trace::CALL_ARG:
            parse_arg(call);
            break;
         case Trace::CALL_RET:
            call->ret = parse_value();
            break;
         default:
            std::cerr << "error: unknown call detail " << c << "\n";
            assert(0);
            /* fallthrough */
         case -1:
            return;
         }
      } while(true);
   }

   void parse_arg(Call *call) {
      unsigned index = read_uint();
      Value *value = parse_value();
      if (index >= call->args.size()) {
          call->args.resize(index + 1);
      }
      call->args[index] = value;
   }

   Value *parse_value(void) {
      int c;
      c = read_byte();
      switch(c) {
      case Trace::TYPE_NULL:
         return new Null;
      case Trace::TYPE_FALSE:
         return new Bool(false);
      case Trace::TYPE_TRUE:
         return new Bool(true);
      case Trace::TYPE_SINT:
         return parse_sint();
      case Trace::TYPE_UINT:
         return parse_uint();
      case Trace::TYPE_FLOAT:
         return parse_float();
      case Trace::TYPE_DOUBLE:
         return parse_double();
      case Trace::TYPE_STRING:
         return parse_string();
      case Trace::TYPE_ENUM:
         return parse_enum();
      case Trace::TYPE_BITMASK:
         return parse_bitmask();
      case Trace::TYPE_ARRAY:
         return parse_array();
      case Trace::TYPE_STRUCT:
         return parse_struct();
      case Trace::TYPE_BLOB:
         return parse_blob();
      case Trace::TYPE_OPAQUE:
         return parse_opaque();
      default:
         std::cerr << "error: unknown type " << c << "\n";
         assert(0);
         return NULL;
      }
   }

   Value *parse_sint() {
      return new SInt(-(signed long long)read_uint());
   }

   Value *parse_uint() {
      return new UInt(read_uint());
   }

   Value *parse_float() {
      float value;
      gzread(file, &value, sizeof value);
      return new Float(value);
   }

   Value *parse_double() {
      double value;
      gzread(file, &value, sizeof value);
      return new Float(value);
   }

   Value *parse_string() {
      return new String(read_string());
   }

   Value *parse_enum() {
      size_t id = read_uint();
      Enum *sig = lookup(enums, id);
      if (!sig) {
          std::string name = read_string();
          Value *value = parse_value();
          sig = new Enum(name, value);
          enums[id] = sig;
      }
      assert(sig);
      return sig;
   }

   Value *parse_bitmask() {
      size_t id = read_uint();
      Bitmask::Signature *sig = lookup(bitmasks, id);
      if (!sig) {
          size_t size = read_uint();
          sig = new Bitmask::Signature(size);
          for (Bitmask::Signature::iterator it = sig->begin(); it != sig->end(); ++it) {
              it->first = read_string();
              it->second = read_uint();
              assert(it->second);
          }
          bitmasks[id] = sig;
      }
      assert(sig);

      unsigned long long value = read_uint();

      return new Bitmask(sig, value);
   }

   Value *parse_array(void) {
      size_t len = read_uint();
      Array *array = new Array(len);
      for (size_t i = 0; i < len; ++i) {
         array->values[i] = parse_value();
      }
      return array;
   }

   Value *parse_blob(void) {
      size_t size = read_uint();
      Blob *blob = new Blob(size);
      if (size) {
          gzread(file, blob->buf, size);
      }
      return blob;
   }

   Value *parse_struct() {
      size_t id = read_uint();

      Struct::Signature *sig = lookup(structs, id);
      if (!sig) {
          sig = new Struct::Signature;
          sig->name = read_string();
          unsigned size = read_uint();
          for (unsigned i = 0; i < size; ++i) {
              sig->member_names.push_back(read_string());
          }
          structs[id] = sig;
      }
      assert(sig);

      Struct *value = new Struct(sig);

      for (size_t i = 0; i < sig->member_names.size(); ++i) {
          value->members[i] = parse_value();
      }

      return value;
   }

   Value *parse_opaque() {
      unsigned long long addr;
      addr = read_uint();
      /* XXX */
      return new UInt(addr);
   }

   std::string read_string(void) {
      size_t len = read_uint();
      if (!len) {
         return std::string();
      }
      char * buf = new char[len];
      gzread(file, buf, len);
      std::string value(buf, len);
      delete [] buf;
#if TRACE_VERBOSE
      std::cerr << "\tSTRING \"" << value << "\"\n";
#endif
      return value;
   }

   unsigned long long read_uint(void) {
      unsigned long long value = 0;
      int c;
      unsigned shift = 0;
      do {
         c = gzgetc(file);
         if (c == -1) {
            break;
         }
         value |= (unsigned long long)(c & 0x7f) << shift;
         shift += 7;
      } while(c & 0x80);
#if TRACE_VERBOSE
      std::cerr << "\tUINT " << value << "\n";
#endif
      return value;
   }

   int read_byte(void) {
      int c = gzgetc(file);
#if TRACE_VERBOSE
      if (c < 0)
         std::cerr << "\tEOF" << "\n";
      else
         std::cerr << "\tBYTE 0x" << std::hex << c << std::dec << "\n";
#endif
      return c;
   }
};

} /* namespace Trace */

#endif /* _TRACE_PARSER_HPP_ */
<|endoftext|>


<commit_before>// Copyright (c) 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// A tool to dump HTML5 filesystem from CUI.
//
// Usage:
//
// ./out/Release/dump_file_system [options] <filesystem dir> [origin]...
//
// If no origin is specified, this dumps all origins in the profile dir.
//
// Available options:
//
// -t : dumps temporary files instead of persistent.
// -s : dumps syncable files instead of persistent.
// -l : more information will be displayed.
//
// The format of -l option is:
//
// === ORIGIN origin_name origin_dir ===
// file_name file_id file_size file_content_path
// ...
//
// where file_name has a trailing slash, file_size is the number of
// children, and file_content_path is empty if the file is a directory.
//

#include <stdio.h>
#include <stdlib.h>

#include <stack>
#include <string>
#include <utility>
#include <vector>

#include "base/file_util.h"
#include "base/files/file_path.h"
#include "base/format_macros.h"
#include "base/strings/stringprintf.h"
#include "webkit/browser/fileapi/obfuscated_file_util.h"
#include "webkit/browser/fileapi/sandbox_directory_database.h"
#include "webkit/browser/fileapi/sandbox_file_system_backend.h"
#include "webkit/browser/fileapi/sandbox_origin_database.h"
#include "webkit/common/fileapi/file_system_types.h"
#include "webkit/common/fileapi/file_system_util.h"

namespace {

bool g_opt_long;
fileapi::FileSystemType g_opt_fs_type = fileapi::kFileSystemTypePersistent;

void ShowMessageAndExit(const std::string& msg) {
  fprintf(stderr, "%s\n", msg.c_str());
  exit(EXIT_FAILURE);
}

void ShowUsageAndExit(const std::string& arg0) {
  ShowMessageAndExit(
      "Usage: " + arg0 +
      " [-l] [-t] [-s] <filesystem dir> [origin]...");
}

}  // namespace

namespace fileapi {

static void DumpDirectoryTree(const std::string& origin_name,
                              base::FilePath origin_dir) {
  origin_dir = origin_dir.Append(
      ObfuscatedFileUtil::GetDirectoryNameForType(g_opt_fs_type));

  printf("=== ORIGIN %s %s ===\n",
         origin_name.c_str(), FilePathToString(origin_dir).c_str());

  if (!base::DirectoryExists(origin_dir))
    return;

  SandboxDirectoryDatabase directory_db(origin_dir);
  SandboxDirectoryDatabase::FileId root_id;
  if (!directory_db.GetFileWithPath(StringToFilePath("/"), &root_id))
    return;

  std::stack<std::pair<SandboxDirectoryDatabase::FileId,
                       std::string> > paths;
  paths.push(std::make_pair(root_id, ""));
  while (!paths.empty()) {
    SandboxDirectoryDatabase::FileId id = paths.top().first;
    const std::string dirname = paths.top().second;
    paths.pop();

    SandboxDirectoryDatabase::FileInfo info;
    if (!directory_db.GetFileInfo(id, &info)) {
      ShowMessageAndExit(base::StringPrintf("GetFileInfo failed for %"PRId64,
                                            id));
    }

    const std::string name =
        dirname + "/" + FilePathToString(base::FilePath(info.name));
    std::vector<SandboxDirectoryDatabase::FileId> children;
    if (info.is_directory()) {
      if (!directory_db.ListChildren(id, &children)) {
        ShowMessageAndExit(base::StringPrintf(
            "ListChildren failed for %s (%"PRId64")",
            info.name.c_str(), id));
      }

      for (size_t j = children.size(); j; j--)
        paths.push(make_pair(children[j-1], name));
    }

    // +1 for the leading extra slash.
    const char* display_name = name.c_str() + 1;
    const char* directory_suffix = info.is_directory() ? "/" : "";
    if (g_opt_long) {
      int64 size;
      if (info.is_directory()) {
        size = static_cast<int64>(children.size());
      } else {
        file_util::GetFileSize(origin_dir.Append(info.data_path), &size);
      }
      // TODO(hamaji): Modification time?
      printf("%s%s %"PRId64" %"PRId64" %s\n",
             display_name,
             directory_suffix,
             id,
             size,
             FilePathToString(info.data_path).c_str());
    } else {
      printf("%s%s\n", display_name, directory_suffix);
    }
  }
}

static void DumpOrigin(const base::FilePath& file_system_dir,
                       const std::string& origin_name) {
  SandboxOriginDatabase origin_db(file_system_dir);
  base::FilePath origin_dir;
  if (!origin_db.HasOriginPath(origin_name)) {
    ShowMessageAndExit("Origin " + origin_name + " is not in " +
                       FilePathToString(file_system_dir));
  }

  if (!origin_db.GetPathForOrigin(origin_name, &origin_dir)) {
    ShowMessageAndExit("Failed to get path of origin " + origin_name +
                       " in " + FilePathToString(file_system_dir));
  }
  DumpDirectoryTree(origin_name, file_system_dir.Append(origin_dir));
}

static void DumpFileSystem(const base::FilePath& file_system_dir) {
  SandboxOriginDatabase origin_db(file_system_dir);
  std::vector<SandboxOriginDatabase::OriginRecord> origins;
  origin_db.ListAllOrigins(&origins);
  for (size_t i = 0; i < origins.size(); i++) {
    const SandboxOriginDatabase::OriginRecord& origin = origins[i];
    DumpDirectoryTree(origin.origin, file_system_dir.Append(origin.path));
    puts("");
  }
}

}  // namespace fileapi

int main(int argc, char* argv[]) {
  const char* arg0 = argv[0];
  std::string username = "Default";
  while (true) {
    if (argc < 2)
      ShowUsageAndExit(arg0);

    if (std::string(argv[1]) == "-l") {
      g_opt_long = true;
      argc--;
      argv++;
    } else if (std::string(argv[1]) == "-t") {
      g_opt_fs_type = fileapi::kFileSystemTypeTemporary;
      argc--;
      argv++;
    } else if (std::string(argv[1]) == "-s") {
      g_opt_fs_type = fileapi::kFileSystemTypeSyncable;
      argc--;
      argv++;
    } else {
      break;
    }
  }

  if (argc < 2)
    ShowUsageAndExit(arg0);

  const base::FilePath file_system_dir = fileapi::StringToFilePath(argv[1]);
  if (!base::DirectoryExists(file_system_dir)) {
    ShowMessageAndExit(fileapi::FilePathToString(file_system_dir) +
                       " is not a filesystem directory");
  }

  if (argc == 2) {
    fileapi::DumpFileSystem(file_system_dir);
  } else {
    for (int i = 2; i < argc; i++) {
      fileapi::DumpOrigin(file_system_dir, argv[i]);
    }
  }
  return 0;
}
<commit_msg>[FileAPI] Fix dump tool build failure<commit_after>// Copyright (c) 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// A tool to dump HTML5 filesystem from CUI.
//
// Usage:
//
// ./out/Release/dump_file_system [options] <filesystem dir> [origin]...
//
// If no origin is specified, this dumps all origins in the profile dir.
//
// Available options:
//
// -t : dumps temporary files instead of persistent.
// -s : dumps syncable files instead of persistent.
// -l : more information will be displayed.
//
// The format of -l option is:
//
// === ORIGIN origin_name origin_dir ===
// file_name file_id file_size file_content_path
// ...
//
// where file_name has a trailing slash, file_size is the number of
// children, and file_content_path is empty if the file is a directory.
//

#include <stdio.h>
#include <stdlib.h>

#include <stack>
#include <string>
#include <utility>
#include <vector>

#include "base/file_util.h"
#include "base/files/file_path.h"
#include "base/format_macros.h"
#include "base/strings/stringprintf.h"
#include "webkit/browser/fileapi/obfuscated_file_util.h"
#include "webkit/browser/fileapi/sandbox_directory_database.h"
#include "webkit/browser/fileapi/sandbox_file_system_backend.h"
#include "webkit/browser/fileapi/sandbox_origin_database.h"
#include "webkit/common/fileapi/file_system_types.h"
#include "webkit/common/fileapi/file_system_util.h"

namespace {

bool g_opt_long;
const char* g_opt_fs_type = "p";

void ShowMessageAndExit(const std::string& msg) {
  fprintf(stderr, "%s\n", msg.c_str());
  exit(EXIT_FAILURE);
}

void ShowUsageAndExit(const std::string& arg0) {
  ShowMessageAndExit(
      "Usage: " + arg0 +
      " [-l] [-t] [-s] <filesystem dir> [origin]...");
}

}  // namespace

namespace fileapi {

static void DumpDirectoryTree(const std::string& origin_name,
                              base::FilePath origin_dir) {
  origin_dir = origin_dir.Append(g_opt_fs_type);

  printf("=== ORIGIN %s %s ===\n",
         origin_name.c_str(), FilePathToString(origin_dir).c_str());

  if (!base::DirectoryExists(origin_dir))
    return;

  SandboxDirectoryDatabase directory_db(origin_dir);
  SandboxDirectoryDatabase::FileId root_id;
  if (!directory_db.GetFileWithPath(StringToFilePath("/"), &root_id))
    return;

  std::stack<std::pair<SandboxDirectoryDatabase::FileId,
                       std::string> > paths;
  paths.push(std::make_pair(root_id, ""));
  while (!paths.empty()) {
    SandboxDirectoryDatabase::FileId id = paths.top().first;
    const std::string dirname = paths.top().second;
    paths.pop();

    SandboxDirectoryDatabase::FileInfo info;
    if (!directory_db.GetFileInfo(id, &info)) {
      ShowMessageAndExit(base::StringPrintf("GetFileInfo failed for %"PRId64,
                                            id));
    }

    const std::string name =
        dirname + "/" + FilePathToString(base::FilePath(info.name));
    std::vector<SandboxDirectoryDatabase::FileId> children;
    if (info.is_directory()) {
      if (!directory_db.ListChildren(id, &children)) {
        ShowMessageAndExit(base::StringPrintf(
            "ListChildren failed for %s (%"PRId64")",
            info.name.c_str(), id));
      }

      for (size_t j = children.size(); j; j--)
        paths.push(make_pair(children[j-1], name));
    }

    // +1 for the leading extra slash.
    const char* display_name = name.c_str() + 1;
    const char* directory_suffix = info.is_directory() ? "/" : "";
    if (g_opt_long) {
      int64 size;
      if (info.is_directory()) {
        size = static_cast<int64>(children.size());
      } else {
        file_util::GetFileSize(origin_dir.Append(info.data_path), &size);
      }
      // TODO(hamaji): Modification time?
      printf("%s%s %"PRId64" %"PRId64" %s\n",
             display_name,
             directory_suffix,
             id,
             size,
             FilePathToString(info.data_path).c_str());
    } else {
      printf("%s%s\n", display_name, directory_suffix);
    }
  }
}

static void DumpOrigin(const base::FilePath& file_system_dir,
                       const std::string& origin_name) {
  SandboxOriginDatabase origin_db(file_system_dir);
  base::FilePath origin_dir;
  if (!origin_db.HasOriginPath(origin_name)) {
    ShowMessageAndExit("Origin " + origin_name + " is not in " +
                       FilePathToString(file_system_dir));
  }

  if (!origin_db.GetPathForOrigin(origin_name, &origin_dir)) {
    ShowMessageAndExit("Failed to get path of origin " + origin_name +
                       " in " + FilePathToString(file_system_dir));
  }
  DumpDirectoryTree(origin_name, file_system_dir.Append(origin_dir));
}

static void DumpFileSystem(const base::FilePath& file_system_dir) {
  SandboxOriginDatabase origin_db(file_system_dir);
  std::vector<SandboxOriginDatabase::OriginRecord> origins;
  origin_db.ListAllOrigins(&origins);
  for (size_t i = 0; i < origins.size(); i++) {
    const SandboxOriginDatabase::OriginRecord& origin = origins[i];
    DumpDirectoryTree(origin.origin, file_system_dir.Append(origin.path));
    puts("");
  }
}

}  // namespace fileapi

int main(int argc, char* argv[]) {
  const char* arg0 = argv[0];
  std::string username = "Default";
  while (true) {
    if (argc < 2)
      ShowUsageAndExit(arg0);

    if (std::string(argv[1]) == "-l") {
      g_opt_long = true;
      argc--;
      argv++;
    } else if (std::string(argv[1]) == "-t") {
      g_opt_fs_type = "t";
      argc--;
      argv++;
    } else if (std::string(argv[1]) == "-s") {
      g_opt_fs_type = "s";
      argc--;
      argv++;
    } else {
      break;
    }
  }

  if (argc < 2)
    ShowUsageAndExit(arg0);

  const base::FilePath file_system_dir = fileapi::StringToFilePath(argv[1]);
  if (!base::DirectoryExists(file_system_dir)) {
    ShowMessageAndExit(fileapi::FilePathToString(file_system_dir) +
                       " is not a filesystem directory");
  }

  if (argc == 2) {
    fileapi::DumpFileSystem(file_system_dir);
  } else {
    for (int i = 2; i < argc; i++) {
      fileapi::DumpOrigin(file_system_dir, argv[i]);
    }
  }
  return 0;
}
<|endoftext|>


<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "config.h"

#include "base/compiler_specific.h"

MSVC_PUSH_WARNING_LEVEL(0);
#include "PlatformString.h"
#include "RegularExpression.h"
MSVC_POP_WARNING();

#include "testing/gtest/include/gtest/gtest.h"
#include "webkit/glue/glue_util.h"

using std::wstring;
using webkit_glue::StdWStringToString;
using WebCore::RegularExpression;
using WebCore::String;

namespace {

class RegexTest : public testing::Test {
};

struct Match {
  const std::wstring text;
  const int position;
  const int length;
};

}  // namespace

TEST(RegexTest, Basic) {
  // Just make sure we're not completely broken.
  const String pattern("the quick brown fox");
  RegularExpression regex(pattern, WebCore::TextCaseSensitive);
  EXPECT_EQ(0, regex.match("the quick brown fox"));
  EXPECT_EQ(1, regex.match(" the quick brown fox"));
  EXPECT_EQ(3, regex.match("foothe quick brown foxbar"));

  EXPECT_EQ(-1, regex.match("The quick brown FOX"));
  EXPECT_EQ(-1, regex.match("the quick brown fo"));
}

TEST(RegexTest, Unicode) {
  // Make sure we get the right offsets for unicode strings.

  // Test 1
  wstring wstr_pattern(L"\x6240\x6709\x7f51\x9875");
  String pattern = StdWStringToString(wstr_pattern);
  RegularExpression regex(pattern, WebCore::TextCaseInsensitive);

  EXPECT_EQ(0, regex.match(StdWStringToString(wstr_pattern)));
  EXPECT_EQ(1, regex.match(StdWStringToString(
      wstring(L" ") + wstr_pattern)));
  EXPECT_EQ(3, regex.match(StdWStringToString(
      wstring(L"foo") + wstr_pattern + wstring(L"bar"))));
  EXPECT_EQ(4, regex.match(StdWStringToString(
      wstring(L"\x4e2d\x6587\x7f51\x9875") + wstr_pattern)));

  // Test 2, mixed length
  wstr_pattern = L":[ \x2000]+:";
  pattern = StdWStringToString(wstr_pattern);
  regex = RegularExpression(pattern, WebCore::TextCaseInsensitive);

  const Match matches[] = {
    { L":  :", 0, 4 },
    { L"  :    :  ", 2, 6 },
    { L" : \x2000 : ", 1, 5 },
    { L"\x6240\x6709\x7f51\x9875 : \x2000 \x2000 : ", 5, 7 },
    { L"", -1, -1 },
    { L"::", -1, -1 },
  };
  for (size_t i = 0; i < arraysize(matches); ++i) {
    EXPECT_EQ(matches[i].position, regex.match(StdWStringToString(
        wstring(matches[i].text))));
    EXPECT_EQ(matches[i].length, regex.matchedLength());
  }

  // Test 3, empty match
  wstr_pattern = L"|x";
  pattern = StdWStringToString(wstr_pattern);
  regex = RegularExpression(pattern, WebCore::TextCaseInsensitive);

  const Match matches2[] = {
    { L"", 0, 0 },
  };
  for (size_t i = 0; i < arraysize(matches2); ++i) {
    EXPECT_EQ(matches2[i].position, regex.match(StdWStringToString(
        wstring(matches2[i].text))));
    EXPECT_EQ(matches2[i].length, regex.matchedLength());
  }
}
<commit_msg>regular_expression_unittest to only use WebKit API.<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/WebKit/WebKit/chromium/public/WebRegularExpression.h"
#include "third_party/WebKit/WebKit/chromium/public/WebString.h"
#include "third_party/WebKit/WebKit/chromium/public/WebTextCaseSensitivity.h"

using namespace WebKit;

namespace {

class RegexTest : public testing::Test {
};

struct Match {
  const WebUChar* text;
  const int textLength;
  const int matchPosition;
  const int matchLength;
};

void testMatches(const WebRegularExpression& regex,
                 const Match* matches,
                 const size_t nMatches) {

  for (size_t i = 0; i < nMatches; ++i) {
    int matchedLength = matches[i].textLength;
    EXPECT_EQ(matches[i].matchPosition, regex.match(
        WebString(matches[i].text, matches[i].textLength), 0, &matchedLength));
    if (matches[i].matchPosition != -1)
      EXPECT_EQ(matches[i].matchLength, matchedLength);
  }

}

}  // namespace

#define MATCH_DESC(webuchar, matchPosition, matchLength) \
    { webuchar, arraysize(webuchar), matchPosition, matchLength }

TEST(RegexTest, Basic) {
  // Just make sure we're not completely broken.
  WebRegularExpression regex("the quick brown fox", WebTextCaseSensitive);
  EXPECT_EQ(0, regex.match("the quick brown fox"));
  EXPECT_EQ(1, regex.match(" the quick brown fox"));
  EXPECT_EQ(3, regex.match("foothe quick brown foxbar"));

  EXPECT_EQ(-1, regex.match("The quick brown FOX"));
  EXPECT_EQ(-1, regex.match("the quick brown fo"));
}

TEST(RegexTest, Unicode) {
  // Make sure we get the right offsets for unicode strings.
  WebUChar pattern[] = {L'\x6240', L'\x6709', L'\x7f51', L'\x9875'};
  WebRegularExpression regex(WebString(pattern, arraysize(pattern)),
                             WebTextCaseInsensitive);

  WebUChar text1[] = {L'\x6240', L'\x6709', L'\x7f51', L'\x9875'};
  WebUChar text2[] = {L' ', L'\x6240', L'\x6709', L'\x7f51', L'\x9875'};
  WebUChar text3[] = {L'f', L'o', L'o', L'\x6240', L'\x6709', L'\x7f51', L'\x9875', L'b', L'a', L'r'};
  WebUChar text4[] = {L'\x4e2d', L'\x6587', L'\x7f51',  L'\x9875', L'\x6240', L'\x6709', L'\x7f51', L'\x9875'};

  const Match matches[] = {
    MATCH_DESC(text1, 0, 4),
    MATCH_DESC(text2, 1, 4),
    MATCH_DESC(text3, 3, 4),
    MATCH_DESC(text4, 4, 4),
  };

  testMatches(regex, matches, arraysize(matches));
}

TEST(RegexTest, UnicodeMixedLength) {
  WebUChar pattern[] = {L':', L'[', L' ', L'\x2000', L']', L'+', L':'};
  WebRegularExpression regex(WebString(pattern, arraysize(pattern)),
                             WebTextCaseInsensitive);

  WebUChar text1[] = {L':', L' ', L' ', L':'};
  WebUChar text2[] = {L' ', L' ', L':', L' ', L' ', L' ', L' ', L':', L' ', L' '};
  WebUChar text3[] = {L' ', L':', L' ', L'\x2000', L' ', L':', L' '};
  WebUChar text4[] = {L'\x6240', L'\x6709', L'\x7f51', L'\x9875', L' ', L':', L' ', L'\x2000', L' ', L'\x2000', L' ', L':', L' '};
  WebUChar text5[] = {L' '};
  WebUChar text6[] = {L':', L':'};

  const Match matches[] = {
    MATCH_DESC(text1, 0, 4),
    MATCH_DESC(text2, 2, 6),
    MATCH_DESC(text3, 1, 5),
    MATCH_DESC(text4, 5, 7),
    MATCH_DESC(text5, -1, -1),
    MATCH_DESC(text6, -1, -1),
  };

  testMatches(regex, matches, arraysize(matches));
}

TEST(RegexTest, EmptyMatch) {
  WebRegularExpression regex("|x", WebTextCaseInsensitive);
  int matchedLength = 0;
  EXPECT_EQ(0, regex.match("", 0, &matchedLength));
  EXPECT_EQ(0, matchedLength);
}
<|endoftext|>


<commit_before>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/at_exit.h"
#include "base/basictypes.h"
#include "base/command_line.h"
#include "base/event_recorder.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/i18n/icu_util.h"
#include "base/memory_debug.h"
#include "base/message_loop.h"
#include "base/path_service.h"
#include "base/process_util.h"
#include "base/rand_util.h"
#include "base/stats_table.h"
#include "base/string_util.h"
#include "base/sys_info.h"
#include "base/trace_event.h"
#include "net/base/cookie_monster.h"
#include "net/base/net_module.h"
#include "net/base/net_util.h"
#include "net/http/http_cache.h"
#include "net/socket/ssl_test_util.h"
#include "net/url_request/url_request_context.h"
#include "third_party/WebKit/WebKit/chromium/public/WebKit.h"
#include "third_party/WebKit/WebKit/chromium/public/WebScriptController.h"
#include "webkit/glue/webkit_glue.h"
#include "webkit/glue/window_open_disposition.h"
#include "webkit/extensions/v8/gc_extension.h"
#include "webkit/extensions/v8/heap_profiler_extension.h"
#include "webkit/extensions/v8/playback_extension.h"
#include "webkit/extensions/v8/profiler_extension.h"
#include "webkit/tools/test_shell/simple_resource_loader_bridge.h"
#include "webkit/tools/test_shell/test_shell.h"
#include "webkit/tools/test_shell/test_shell_platform_delegate.h"
#include "webkit/tools/test_shell/test_shell_request_context.h"
#include "webkit/tools/test_shell/test_shell_switches.h"
#include "webkit/tools/test_shell/test_shell_webkit_init.h"

static const size_t kPathBufSize = 2048;

using WebKit::WebScriptController;

namespace {

// StatsTable initialization parameters.
const char* const kStatsFilePrefix = "testshell_";
int kStatsFileThreads = 20;
int kStatsFileCounters = 200;

void RemoveSharedMemoryFile(std::string& filename) {
  // Stats uses SharedMemory under the hood. On posix, this results in a file
  // on disk.
#if defined(OS_POSIX)
  base::SharedMemory memory;
  memory.Delete(UTF8ToWide(filename));
#endif
}

}  // namespace

int main(int argc, char* argv[]) {
  base::EnableInProcessStackDumping();
  base::EnableTerminationOnHeapCorruption();

  // Some tests may use base::Singleton<>, thus we need to instanciate
  // the AtExitManager or else we will leak objects.
  base::AtExitManager at_exit_manager;

  TestShellPlatformDelegate::PreflightArgs(&argc, &argv);
  CommandLine::Init(argc, argv);
  const CommandLine& parsed_command_line = *CommandLine::ForCurrentProcess();

  TestShellPlatformDelegate platform(parsed_command_line);

  if (parsed_command_line.HasSwitch(test_shell::kStartupDialog))
    TestShell::ShowStartupDebuggingDialog();

  if (parsed_command_line.HasSwitch(test_shell::kCheckLayoutTestSystemDeps)) {
    exit(platform.CheckLayoutTestSystemDependencies() ? 0 : 1);
  }

  // Allocate a message loop for this thread.  Although it is not used
  // directly, its constructor sets up some necessary state.
  MessageLoopForUI main_message_loop;

  bool suppress_error_dialogs = (
       base::SysInfo::HasEnvVar(L"CHROME_HEADLESS") ||
       parsed_command_line.HasSwitch(test_shell::kNoErrorDialogs) ||
       parsed_command_line.HasSwitch(test_shell::kLayoutTests));
  bool layout_test_mode =
      parsed_command_line.HasSwitch(test_shell::kLayoutTests);
  bool ux_theme = parsed_command_line.HasSwitch(test_shell::kUxTheme);
  bool classic_theme =
      parsed_command_line.HasSwitch(test_shell::kClassicTheme);
#if defined(OS_WIN)
  bool generic_theme = (layout_test_mode && !ux_theme && !classic_theme) ||
      parsed_command_line.HasSwitch(test_shell::kGenericTheme);
#else
  // Stop compiler warnings about unused variables.
  ux_theme = ux_theme;
#endif

  bool enable_gp_fault_error_box = false;
  enable_gp_fault_error_box =
      parsed_command_line.HasSwitch(test_shell::kGPFaultErrorBox);
  TestShell::InitLogging(suppress_error_dialogs,
                         layout_test_mode,
                         enable_gp_fault_error_box);

  // Initialize WebKit for this scope.
  TestShellWebKitInit test_shell_webkit_init(layout_test_mode);

  // Suppress abort message in v8 library in debugging mode (but not
  // actually under a debugger).  V8 calls abort() when it hits
  // assertion errors.
  if (suppress_error_dialogs) {
    platform.SuppressErrorReporting();
  }

  if (parsed_command_line.HasSwitch(test_shell::kEnableTracing))
    base::TraceLog::StartTracing();

  net::HttpCache::Mode cache_mode = net::HttpCache::NORMAL;

  // This is a special mode where JS helps the browser implement
  // playback/record mode.  Generally, in this mode, some functions
  // of client-side randomness are removed.  For example, in
  // this mode Math.random() and Date.getTime() may not return
  // values which vary.
  bool playback_mode =
    parsed_command_line.HasSwitch(test_shell::kPlaybackMode);
  bool record_mode =
    parsed_command_line.HasSwitch(test_shell::kRecordMode);

  if (playback_mode)
    cache_mode = net::HttpCache::PLAYBACK;
  else if (record_mode)
    cache_mode = net::HttpCache::RECORD;

  if (layout_test_mode ||
      parsed_command_line.HasSwitch(test_shell::kEnableFileCookies))
    net::CookieMonster::EnableFileScheme();

  FilePath cache_path =
      parsed_command_line.GetSwitchValuePath(test_shell::kCacheDir);
  // If the cache_path is empty and it's layout_test_mode, leave it empty
  // so we use an in-memory cache. This makes running multiple test_shells
  // in parallel less flaky.
  if (cache_path.empty() && !layout_test_mode) {
    PathService::Get(base::DIR_EXE, &cache_path);
    cache_path = cache_path.AppendASCII("cache");
  }

  // Initializing with a default context, which means no on-disk cookie DB,
  // and no support for directory listings.
  SimpleResourceLoaderBridge::Init(
      new TestShellRequestContext(cache_path, cache_mode, layout_test_mode));

  // Load ICU data tables
  icu_util::Initialize();

  // Config the network module so it has access to a limited set of resources.
  net::NetModule::SetResourceProvider(TestShell::NetResourceProvider);

  // On Linux, load the test root certificate.
  net::TestServerLauncher ssl_util;
  ssl_util.LoadTestRootCert();

  platform.InitializeGUI();

  TestShell::InitializeTestShell(layout_test_mode);

  if (parsed_command_line.HasSwitch(test_shell::kAllowScriptsToCloseWindows))
    TestShell::SetAllowScriptsToCloseWindows();

  // Disable user themes for layout tests so pixel tests are consistent.
#if defined(OS_WIN)
  TestShellWebTheme::Engine engine;
#endif
  if (classic_theme)
    platform.SelectUnifiedTheme();
#if defined(OS_WIN)
  if (generic_theme)
    test_shell_webkit_init.SetThemeEngine(&engine);
#endif

  if (parsed_command_line.HasSwitch(test_shell::kTestShellTimeOut)) {
    const std::wstring timeout_str = parsed_command_line.GetSwitchValue(
        test_shell::kTestShellTimeOut);
    int timeout_ms =
        static_cast<int>(StringToInt64(WideToUTF16Hack(timeout_str.c_str())));
    if (timeout_ms > 0)
      TestShell::SetFileTestTimeout(timeout_ms);
  }

  // Treat the first loose value as the initial URL to open.
  GURL starting_url;

  // Default to a homepage if we're interactive.
  if (!layout_test_mode) {
    FilePath path;
    PathService::Get(base::DIR_SOURCE_ROOT, &path);
    path = path.AppendASCII("webkit");
    path = path.AppendASCII("data");
    path = path.AppendASCII("test_shell");
    path = path.AppendASCII("index.html");
    starting_url = net::FilePathToFileURL(path);
  }

  std::vector<std::wstring> loose_values = parsed_command_line.GetLooseValues();
  if (loose_values.size() > 0) {
    GURL url(WideToUTF16Hack(loose_values[0]));
    if (url.is_valid()) {
      starting_url = url;
    } else {
      // Treat as a file path
      starting_url =
          net::FilePathToFileURL(FilePath::FromWStringHack(loose_values[0]));
    }
  }

  std::wstring js_flags =
    parsed_command_line.GetSwitchValue(test_shell::kJavaScriptFlags);
  // Test shell always exposes the GC.
  js_flags += L" --expose-gc";
  webkit_glue::SetJavaScriptFlags(js_flags);
  // Expose GCController to JavaScript.
  WebScriptController::registerExtension(extensions_v8::GCExtension::Get());

  if (parsed_command_line.HasSwitch(test_shell::kProfiler)) {
    WebScriptController::registerExtension(
        extensions_v8::ProfilerExtension::Get());
  }

  if (parsed_command_line.HasSwitch(test_shell::kHeapProfiler)) {
    WebScriptController::registerExtension(
        extensions_v8::HeapProfilerExtension::Get());
  }

  // Load and initialize the stats table.  Attempt to construct a somewhat
  // unique name to isolate separate instances from each other.

  // truncate the random # to 32 bits for the benefit of Mac OS X, to
  // avoid tripping over its maximum shared memory segment name length
  std::string stats_filename =
      kStatsFilePrefix + Uint64ToString(base::RandUint64() & 0xFFFFFFFFL);
  RemoveSharedMemoryFile(stats_filename);
  StatsTable *table = new StatsTable(stats_filename,
      kStatsFileThreads,
      kStatsFileCounters);
  StatsTable::set_current(table);

  TestShell* shell;
  if (TestShell::CreateNewWindow(starting_url, &shell)) {
    if (record_mode || playback_mode) {
      platform.SetWindowPositionForRecording(shell);
      WebScriptController::registerExtension(
          extensions_v8::PlaybackExtension::Get());
    }

    shell->Show(WebKit::WebNavigationPolicyNewWindow);

    if (parsed_command_line.HasSwitch(test_shell::kDumpStatsTable))
      shell->DumpStatsTableOnExit();

    bool no_events = parsed_command_line.HasSwitch(test_shell::kNoEvents);
    if ((record_mode || playback_mode) && !no_events) {
      FilePath script_path = cache_path;
      // Create the cache directory in case it doesn't exist.
      file_util::CreateDirectory(cache_path);
      script_path = script_path.AppendASCII("script.log");
      if (record_mode)
        base::EventRecorder::current()->StartRecording(script_path);
      if (playback_mode)
        base::EventRecorder::current()->StartPlayback(script_path);
    }

    if (parsed_command_line.HasSwitch(test_shell::kDebugMemoryInUse)) {
      base::MemoryDebug::SetMemoryInUseEnabled(true);
      // Dump all in use memory at startup
      base::MemoryDebug::DumpAllMemoryInUse();
    }

    // See if we need to run the tests.
    if (layout_test_mode) {
      // Set up for the kind of test requested.
      TestShell::TestParams params;
      if (parsed_command_line.HasSwitch(test_shell::kDumpPixels)) {
        // The pixel test flag also gives the image file name to use.
        params.dump_pixels = true;
        params.pixel_file_name = parsed_command_line.GetSwitchValue(
            test_shell::kDumpPixels);
        if (params.pixel_file_name.size() == 0) {
          fprintf(stderr, "No file specified for pixel tests");
          exit(1);
        }
      }
      if (parsed_command_line.HasSwitch(test_shell::kNoTree)) {
          params.dump_tree = false;
      }

      if (!starting_url.is_valid()) {
        // Watch stdin for URLs.
        char filenameBuffer[kPathBufSize];
        while (fgets(filenameBuffer, sizeof(filenameBuffer), stdin)) {
          // When running layout tests we pass new line separated
          // tests to TestShell. Each line is a space separated list
          // of filename, timeout and expected pixel hash. The timeout
          // and the pixel hash are optional.
          char* newLine = strchr(filenameBuffer, '\n');
          if (newLine)
            *newLine = '\0';
          if (!*filenameBuffer)
            continue;

          params.test_url = strtok(filenameBuffer, " ");

          // Set the current path to the directory that contains the test
          // files. This is because certain test file may use the relative
          // path.
          GURL test_url(params.test_url);
          FilePath test_file_path;
          net::FileURLToFilePath(test_url, &test_file_path);
          file_util::SetCurrentDirectory(test_file_path.DirName());

          int old_timeout_ms = TestShell::GetLayoutTestTimeout();

          char* timeout = strtok(NULL, " ");
          if (timeout) {
            TestShell::SetFileTestTimeout(atoi(timeout));
            char* pixel_hash = strtok(NULL, " ");
            if (pixel_hash)
              params.pixel_hash = pixel_hash;
          }

          if (!TestShell::RunFileTest(params))
            break;

          TestShell::SetFileTestTimeout(old_timeout_ms);
        }
      } else {
        // TODO(ojan): Provide a way for run-singly tests to pass
        // in a hash and then set params.pixel_hash here.
        params.test_url = WideToUTF8(loose_values[0]);
        TestShell::RunFileTest(params);
      }

      shell->CallJSGC();
      shell->CallJSGC();

      // When we finish the last test, cleanup the LayoutTestController.
      // It may have references to not-yet-cleaned up windows.  By
      // cleaning up here we help purify reports.
      shell->ResetTestController();

      // Flush any remaining messages before we kill ourselves.
      // http://code.google.com/p/chromium/issues/detail?id=9500
      MessageLoop::current()->RunAllPending();

      delete shell;
    } else {
      MessageLoop::current()->Run();
    }

    // Flush any remaining messages.  This ensures that any accumulated
    // Task objects get destroyed before we exit, which avoids noise in
    // purify leak-test results.
    MessageLoop::current()->RunAllPending();

    if (record_mode)
      base::EventRecorder::current()->StopRecording();
    if (playback_mode)
      base::EventRecorder::current()->StopPlayback();
  }

  TestShell::ShutdownTestShell();
  TestShell::CleanupLogging();

  // Tear down shared StatsTable; prevents unit_tests from leaking it.
  StatsTable::set_current(NULL);
  delete table;
  RemoveSharedMemoryFile(stats_filename);

  return 0;
}
<commit_msg>Fix double free on exit in test shell.<commit_after>// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/at_exit.h"
#include "base/basictypes.h"
#include "base/command_line.h"
#include "base/event_recorder.h"
#include "base/file_path.h"
#include "base/file_util.h"
#include "base/i18n/icu_util.h"
#include "base/memory_debug.h"
#include "base/message_loop.h"
#include "base/path_service.h"
#include "base/process_util.h"
#include "base/rand_util.h"
#include "base/stats_table.h"
#include "base/string_util.h"
#include "base/sys_info.h"
#include "base/trace_event.h"
#include "net/base/cookie_monster.h"
#include "net/base/net_module.h"
#include "net/base/net_util.h"
#include "net/http/http_cache.h"
#include "net/socket/ssl_test_util.h"
#include "net/url_request/url_request_context.h"
#include "third_party/WebKit/WebKit/chromium/public/WebKit.h"
#include "third_party/WebKit/WebKit/chromium/public/WebScriptController.h"
#include "webkit/glue/webkit_glue.h"
#include "webkit/glue/window_open_disposition.h"
#include "webkit/extensions/v8/gc_extension.h"
#include "webkit/extensions/v8/heap_profiler_extension.h"
#include "webkit/extensions/v8/playback_extension.h"
#include "webkit/extensions/v8/profiler_extension.h"
#include "webkit/tools/test_shell/simple_resource_loader_bridge.h"
#include "webkit/tools/test_shell/test_shell.h"
#include "webkit/tools/test_shell/test_shell_platform_delegate.h"
#include "webkit/tools/test_shell/test_shell_request_context.h"
#include "webkit/tools/test_shell/test_shell_switches.h"
#include "webkit/tools/test_shell/test_shell_webkit_init.h"

static const size_t kPathBufSize = 2048;

using WebKit::WebScriptController;

namespace {

// StatsTable initialization parameters.
const char* const kStatsFilePrefix = "testshell_";
int kStatsFileThreads = 20;
int kStatsFileCounters = 200;

void RemoveSharedMemoryFile(std::string& filename) {
  // Stats uses SharedMemory under the hood. On posix, this results in a file
  // on disk.
#if defined(OS_POSIX)
  base::SharedMemory memory;
  memory.Delete(UTF8ToWide(filename));
#endif
}

}  // namespace

int main(int argc, char* argv[]) {
  base::EnableInProcessStackDumping();
  base::EnableTerminationOnHeapCorruption();

  // Some tests may use base::Singleton<>, thus we need to instanciate
  // the AtExitManager or else we will leak objects.
  base::AtExitManager at_exit_manager;

  TestShellPlatformDelegate::PreflightArgs(&argc, &argv);
  CommandLine::Init(argc, argv);
  const CommandLine& parsed_command_line = *CommandLine::ForCurrentProcess();

  TestShellPlatformDelegate platform(parsed_command_line);

  if (parsed_command_line.HasSwitch(test_shell::kStartupDialog))
    TestShell::ShowStartupDebuggingDialog();

  if (parsed_command_line.HasSwitch(test_shell::kCheckLayoutTestSystemDeps)) {
    exit(platform.CheckLayoutTestSystemDependencies() ? 0 : 1);
  }

  // Allocate a message loop for this thread.  Although it is not used
  // directly, its constructor sets up some necessary state.
  MessageLoopForUI main_message_loop;

  bool suppress_error_dialogs = (
       base::SysInfo::HasEnvVar(L"CHROME_HEADLESS") ||
       parsed_command_line.HasSwitch(test_shell::kNoErrorDialogs) ||
       parsed_command_line.HasSwitch(test_shell::kLayoutTests));
  bool layout_test_mode =
      parsed_command_line.HasSwitch(test_shell::kLayoutTests);
  bool ux_theme = parsed_command_line.HasSwitch(test_shell::kUxTheme);
  bool classic_theme =
      parsed_command_line.HasSwitch(test_shell::kClassicTheme);
#if defined(OS_WIN)
  bool generic_theme = (layout_test_mode && !ux_theme && !classic_theme) ||
      parsed_command_line.HasSwitch(test_shell::kGenericTheme);
#else
  // Stop compiler warnings about unused variables.
  ux_theme = ux_theme;
#endif

  bool enable_gp_fault_error_box = false;
  enable_gp_fault_error_box =
      parsed_command_line.HasSwitch(test_shell::kGPFaultErrorBox);
  TestShell::InitLogging(suppress_error_dialogs,
                         layout_test_mode,
                         enable_gp_fault_error_box);

  // Initialize WebKit for this scope.
  TestShellWebKitInit test_shell_webkit_init(layout_test_mode);

  // Suppress abort message in v8 library in debugging mode (but not
  // actually under a debugger).  V8 calls abort() when it hits
  // assertion errors.
  if (suppress_error_dialogs) {
    platform.SuppressErrorReporting();
  }

  if (parsed_command_line.HasSwitch(test_shell::kEnableTracing))
    base::TraceLog::StartTracing();

  net::HttpCache::Mode cache_mode = net::HttpCache::NORMAL;

  // This is a special mode where JS helps the browser implement
  // playback/record mode.  Generally, in this mode, some functions
  // of client-side randomness are removed.  For example, in
  // this mode Math.random() and Date.getTime() may not return
  // values which vary.
  bool playback_mode =
    parsed_command_line.HasSwitch(test_shell::kPlaybackMode);
  bool record_mode =
    parsed_command_line.HasSwitch(test_shell::kRecordMode);

  if (playback_mode)
    cache_mode = net::HttpCache::PLAYBACK;
  else if (record_mode)
    cache_mode = net::HttpCache::RECORD;

  if (layout_test_mode ||
      parsed_command_line.HasSwitch(test_shell::kEnableFileCookies))
    net::CookieMonster::EnableFileScheme();

  FilePath cache_path =
      parsed_command_line.GetSwitchValuePath(test_shell::kCacheDir);
  // If the cache_path is empty and it's layout_test_mode, leave it empty
  // so we use an in-memory cache. This makes running multiple test_shells
  // in parallel less flaky.
  if (cache_path.empty() && !layout_test_mode) {
    PathService::Get(base::DIR_EXE, &cache_path);
    cache_path = cache_path.AppendASCII("cache");
  }

  // Initializing with a default context, which means no on-disk cookie DB,
  // and no support for directory listings.
  SimpleResourceLoaderBridge::Init(
      new TestShellRequestContext(cache_path, cache_mode, layout_test_mode));

  // Load ICU data tables
  icu_util::Initialize();

  // Config the network module so it has access to a limited set of resources.
  net::NetModule::SetResourceProvider(TestShell::NetResourceProvider);

  // On Linux, load the test root certificate.
  net::TestServerLauncher ssl_util;
  ssl_util.LoadTestRootCert();

  platform.InitializeGUI();

  TestShell::InitializeTestShell(layout_test_mode);

  if (parsed_command_line.HasSwitch(test_shell::kAllowScriptsToCloseWindows))
    TestShell::SetAllowScriptsToCloseWindows();

  // Disable user themes for layout tests so pixel tests are consistent.
#if defined(OS_WIN)
  TestShellWebTheme::Engine engine;
#endif
  if (classic_theme)
    platform.SelectUnifiedTheme();
#if defined(OS_WIN)
  if (generic_theme)
    test_shell_webkit_init.SetThemeEngine(&engine);
#endif

  if (parsed_command_line.HasSwitch(test_shell::kTestShellTimeOut)) {
    const std::wstring timeout_str = parsed_command_line.GetSwitchValue(
        test_shell::kTestShellTimeOut);
    int timeout_ms =
        static_cast<int>(StringToInt64(WideToUTF16Hack(timeout_str.c_str())));
    if (timeout_ms > 0)
      TestShell::SetFileTestTimeout(timeout_ms);
  }

  // Treat the first loose value as the initial URL to open.
  GURL starting_url;

  // Default to a homepage if we're interactive.
  if (!layout_test_mode) {
    FilePath path;
    PathService::Get(base::DIR_SOURCE_ROOT, &path);
    path = path.AppendASCII("webkit");
    path = path.AppendASCII("data");
    path = path.AppendASCII("test_shell");
    path = path.AppendASCII("index.html");
    starting_url = net::FilePathToFileURL(path);
  }

  std::vector<std::wstring> loose_values = parsed_command_line.GetLooseValues();
  if (loose_values.size() > 0) {
    GURL url(WideToUTF16Hack(loose_values[0]));
    if (url.is_valid()) {
      starting_url = url;
    } else {
      // Treat as a file path
      starting_url =
          net::FilePathToFileURL(FilePath::FromWStringHack(loose_values[0]));
    }
  }

  std::wstring js_flags =
    parsed_command_line.GetSwitchValue(test_shell::kJavaScriptFlags);
  // Test shell always exposes the GC.
  js_flags += L" --expose-gc";
  webkit_glue::SetJavaScriptFlags(js_flags);
  // Expose GCController to JavaScript.
  WebScriptController::registerExtension(extensions_v8::GCExtension::Get());

  if (parsed_command_line.HasSwitch(test_shell::kProfiler)) {
    WebScriptController::registerExtension(
        extensions_v8::ProfilerExtension::Get());
  }

  if (parsed_command_line.HasSwitch(test_shell::kHeapProfiler)) {
    WebScriptController::registerExtension(
        extensions_v8::HeapProfilerExtension::Get());
  }

  // Load and initialize the stats table.  Attempt to construct a somewhat
  // unique name to isolate separate instances from each other.

  // truncate the random # to 32 bits for the benefit of Mac OS X, to
  // avoid tripping over its maximum shared memory segment name length
  std::string stats_filename =
      kStatsFilePrefix + Uint64ToString(base::RandUint64() & 0xFFFFFFFFL);
  RemoveSharedMemoryFile(stats_filename);
  StatsTable *table = new StatsTable(stats_filename,
      kStatsFileThreads,
      kStatsFileCounters);
  StatsTable::set_current(table);

  TestShell* shell;
  if (TestShell::CreateNewWindow(starting_url, &shell)) {
    if (record_mode || playback_mode) {
      platform.SetWindowPositionForRecording(shell);
      WebScriptController::registerExtension(
          extensions_v8::PlaybackExtension::Get());
    }

    shell->Show(WebKit::WebNavigationPolicyNewWindow);

    if (parsed_command_line.HasSwitch(test_shell::kDumpStatsTable))
      shell->DumpStatsTableOnExit();

    bool no_events = parsed_command_line.HasSwitch(test_shell::kNoEvents);
    if ((record_mode || playback_mode) && !no_events) {
      FilePath script_path = cache_path;
      // Create the cache directory in case it doesn't exist.
      file_util::CreateDirectory(cache_path);
      script_path = script_path.AppendASCII("script.log");
      if (record_mode)
        base::EventRecorder::current()->StartRecording(script_path);
      if (playback_mode)
        base::EventRecorder::current()->StartPlayback(script_path);
    }

    if (parsed_command_line.HasSwitch(test_shell::kDebugMemoryInUse)) {
      base::MemoryDebug::SetMemoryInUseEnabled(true);
      // Dump all in use memory at startup
      base::MemoryDebug::DumpAllMemoryInUse();
    }

    // See if we need to run the tests.
    if (layout_test_mode) {
      // Set up for the kind of test requested.
      TestShell::TestParams params;
      if (parsed_command_line.HasSwitch(test_shell::kDumpPixels)) {
        // The pixel test flag also gives the image file name to use.
        params.dump_pixels = true;
        params.pixel_file_name = parsed_command_line.GetSwitchValue(
            test_shell::kDumpPixels);
        if (params.pixel_file_name.size() == 0) {
          fprintf(stderr, "No file specified for pixel tests");
          exit(1);
        }
      }
      if (parsed_command_line.HasSwitch(test_shell::kNoTree)) {
          params.dump_tree = false;
      }

      if (!starting_url.is_valid()) {
        // Watch stdin for URLs.
        char filenameBuffer[kPathBufSize];
        while (fgets(filenameBuffer, sizeof(filenameBuffer), stdin)) {
          // When running layout tests we pass new line separated
          // tests to TestShell. Each line is a space separated list
          // of filename, timeout and expected pixel hash. The timeout
          // and the pixel hash are optional.
          char* newLine = strchr(filenameBuffer, '\n');
          if (newLine)
            *newLine = '\0';
          if (!*filenameBuffer)
            continue;

          params.test_url = strtok(filenameBuffer, " ");

          // Set the current path to the directory that contains the test
          // files. This is because certain test file may use the relative
          // path.
          GURL test_url(params.test_url);
          FilePath test_file_path;
          net::FileURLToFilePath(test_url, &test_file_path);
          file_util::SetCurrentDirectory(test_file_path.DirName());

          int old_timeout_ms = TestShell::GetLayoutTestTimeout();

          char* timeout = strtok(NULL, " ");
          if (timeout) {
            TestShell::SetFileTestTimeout(atoi(timeout));
            char* pixel_hash = strtok(NULL, " ");
            if (pixel_hash)
              params.pixel_hash = pixel_hash;
          }

          if (!TestShell::RunFileTest(params))
            break;

          TestShell::SetFileTestTimeout(old_timeout_ms);
        }
      } else {
        // TODO(ojan): Provide a way for run-singly tests to pass
        // in a hash and then set params.pixel_hash here.
        params.test_url = WideToUTF8(loose_values[0]);
        TestShell::RunFileTest(params);
      }

      shell->CallJSGC();
      shell->CallJSGC();

      // When we finish the last test, cleanup the LayoutTestController.
      // It may have references to not-yet-cleaned up windows.  By
      // cleaning up here we help purify reports.
      shell->ResetTestController();

      // Flush any remaining messages before we kill ourselves.
      // http://code.google.com/p/chromium/issues/detail?id=9500
      MessageLoop::current()->RunAllPending();
    } else {
      MessageLoop::current()->Run();
    }

    if (record_mode)
      base::EventRecorder::current()->StopRecording();
    if (playback_mode)
      base::EventRecorder::current()->StopPlayback();
  }

  TestShell::ShutdownTestShell();
  TestShell::CleanupLogging();

  // Tear down shared StatsTable; prevents unit_tests from leaking it.
  StatsTable::set_current(NULL);
  delete table;
  RemoveSharedMemoryFile(stats_filename);

  return 0;
}
<|endoftext|>


<commit_before>/*

Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#include "Extractor.h"

#include "ExtractionContainers.h"
#include "ExtractionNode.h"
#include "ExtractionWay.h"
#include "ExtractorCallbacks.h"
#include "RestrictionParser.h"
#include "ScriptingEnvironment.h"

#include "../Util/GitDescription.h"
#include "../Util/IniFileUtil.h"
#include "../Util/OSRMException.h"
#include "../Util/simple_logger.hpp"
#include "../Util/TimingUtil.h"
#include "../Util/make_unique.hpp"
#include "../typedefs.h"

#include <boost/program_options.hpp>

#include <luabind/luabind.hpp>

#include <tbb/task_scheduler_init.h>

#include <osmium/io/any_input.hpp>

#include <cstdlib>

#include <chrono>
#include <fstream>
#include <iostream>
#include <thread>
#include <unordered_map>

Extractor::Extractor() : requested_num_threads(0), file_has_pbf_format(false)
{
}

int lua_error_callback(lua_State *L) // This is so I can use my own function as an
// exception handler, pcall_log()
{
    luabind::object error_msg(luabind::from_stack(L, -1));
    std::ostringstream error_stream;
    error_stream << error_msg;
    throw OSRMException("ERROR occured in profile script:\n" + error_stream.str());
}

bool Extractor::ParseArguments(int argc, char *argv[])
{
    // declare a group of options that will be allowed only on command line
    boost::program_options::options_description generic_options("Options");
    generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
        "config,c",
        boost::program_options::value<boost::filesystem::path>(&config_file_path)
            ->default_value("extractor.ini"),
        "Path to a configuration file.");

    // declare a group of options that will be allowed both on command line and in config file
    boost::program_options::options_description config_options("Configuration");
    config_options.add_options()("profile,p",
                                 boost::program_options::value<boost::filesystem::path>(
                                     &profile_path)->default_value("profile.lua"),
                                 "Path to LUA routing profile")(
        "threads,t",
        boost::program_options::value<unsigned int>(&requested_num_threads)
            ->default_value(tbb::task_scheduler_init::default_num_threads()),
        "Number of threads to use");

    // hidden options, will be allowed both on command line and in config file, but will not be
    // shown to the user
    boost::program_options::options_description hidden_options("Hidden options");
    hidden_options.add_options()(
        "input,i",
        boost::program_options::value<boost::filesystem::path>(&input_path),
        "Input file in .osm, .osm.bz2 or .osm.pbf format");

    // positional option
    boost::program_options::positional_options_description positional_options;
    positional_options.add("input", 1);

    // combine above options for parsing
    boost::program_options::options_description cmdline_options;
    cmdline_options.add(generic_options).add(config_options).add(hidden_options);

    boost::program_options::options_description config_file_options;
    config_file_options.add(config_options).add(hidden_options);

    boost::program_options::options_description visible_options(
        boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
    visible_options.add(generic_options).add(config_options);

    // parse command line options
    boost::program_options::variables_map option_variables;
    boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
                                      .options(cmdline_options)
                                      .positional(positional_options)
                                      .run(),
                                  option_variables);

    if (option_variables.count("version"))
    {
        SimpleLogger().Write() << g_GIT_DESCRIPTION;
        return false;
    }

    if (option_variables.count("help"))
    {
        SimpleLogger().Write() << visible_options;
        return false;
    }

    boost::program_options::notify(option_variables);

    // parse config file
    if (boost::filesystem::is_regular_file(config_file_path))
    {
        SimpleLogger().Write() << "Reading options from: " << config_file_path.string();
        std::string ini_file_contents = ReadIniFileAndLowerContents(config_file_path);
        std::stringstream config_stream(ini_file_contents);
        boost::program_options::store(parse_config_file(config_stream, config_file_options),
                                      option_variables);
        boost::program_options::notify(option_variables);
    }

    if (!option_variables.count("input"))
    {
        SimpleLogger().Write() << visible_options;
        return false;
    }

    return true;
}

void Extractor::GenerateOutputFilesNames()
{
    output_file_name = input_path.string();
    restriction_file_name = input_path.string();
    timestamp_file_name = input_path.string();
    std::string::size_type pos = output_file_name.find(".osm.bz2");
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".osm.pbf");
        if (pos != std::string::npos)
        {
            file_has_pbf_format = true;
        }
        else
        {
            pos = output_file_name.find(".osm.xml");
        }
    }
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".pbf");
        if (pos != std::string::npos)
        {
            file_has_pbf_format = true;
        }
    }
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".osm");
        if (pos == std::string::npos)
        {
            output_file_name.append(".osrm");
            restriction_file_name.append(".osrm.restrictions");
            timestamp_file_name.append(".osrm.timestamp");
        }
        else
        {
            output_file_name.replace(pos, 5, ".osrm");
            restriction_file_name.replace(pos, 5, ".osrm.restrictions");
            timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
        }
    }
    else
    {
        output_file_name.replace(pos, 8, ".osrm");
        restriction_file_name.replace(pos, 8, ".osrm.restrictions");
        timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
    }
}

int Extractor::Run(int argc, char *argv[])
{
    try
    {
        LogPolicy::GetInstance().Unmute();

        TIMER_START(extracting);

        if (!ParseArguments(argc, argv))
            return 0;

        if (1 > requested_num_threads)
        {
            SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
            return 1;
        }

        if (!boost::filesystem::is_regular_file(input_path))
        {
            SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
                                             << " not found!";
            return 1;
        }

        if (!boost::filesystem::is_regular_file(profile_path))
        {
            SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
                                             << " not found!";
            return 1;
        }

        const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();

        SimpleLogger().Write() << "Input file: " << input_path.filename().string();
        SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
        SimpleLogger().Write() << "Threads: " << requested_num_threads;
        if (recommended_num_threads != requested_num_threads)
        {
            SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
                                             << recommended_num_threads
                                             << "! This setting may have performance side-effects.";
        }

        tbb::task_scheduler_init init(requested_num_threads);

        /*** Setup Scripting Environment ***/
        ScriptingEnvironment scripting_environment(profile_path.string().c_str());

        GenerateOutputFilesNames();

        std::unordered_map<std::string, NodeID> string_map;
        ExtractionContainers extraction_containers;

        string_map[""] = 0;
        auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);

        osmium::io::File infile(input_path.string());
        osmium::io::Reader reader(infile);
        osmium::io::Header header = reader.header();

        unsigned number_of_nodes = 0;
        unsigned number_of_ways = 0;
        unsigned number_of_relations = 0;
        unsigned number_of_others = 0;

        SimpleLogger().Write() << "Parsing in progress..";
        TIMER_START(parsing);

        std::string generator = header.get("generator");
        if (generator.empty())
        {
            generator = "unknown tool";
        }
        SimpleLogger().Write() << "input file generated by " << generator;

        // TODO: write timestamp if non-empty
        std::string timestamp = header.get("osmosis_replication_timestamp");
        if (timestamp.empty())
        {
            timestamp = "n/a";
        }
        SimpleLogger().Write() << "timestamp: " << timestamp;

        boost::filesystem::ofstream timestamp_out(timestamp_file_name);
        timestamp_out.write(timestamp.c_str(), timestamp.length());
        timestamp_out.close();

        lua_State *lua_state = scripting_environment.getLuaState();
        luabind::set_pcall_callback(&lua_error_callback);

        RestrictionParser restriction_parser(scripting_environment);

        ExtractionNode result_node;
        ExtractionWay result_way;

        while (osmium::memory::Buffer buffer = reader.read())
        {
            for (osmium::OSMEntity &entity : buffer)
            {
                switch (entity.type())
                {
                case osmium::item_type::node:
                    ++number_of_nodes;
                    result_node.Clear();
                    luabind::call_function<void>(
                        lua_state,
                        "node_function",
                        boost::cref(static_cast<osmium::Node &>(entity)),
                        boost::ref(result_node));
                    extractor_callbacks->ProcessNode(static_cast<osmium::Node &>(entity),
                                                     result_node);
                    break;
                case osmium::item_type::way:
                    ++number_of_ways;
                    result_way.Clear();
                    luabind::call_function<void>(
                        lua_state,
                        "way_function",
                        boost::cref(static_cast<osmium::Way &>(entity)),
                        boost::ref(result_way));
                    extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity),
                                                    result_way);
                    break;
                case osmium::item_type::relation:
                    ++number_of_relations;
                    extractor_callbacks->ProcessRestriction(
                        restriction_parser.TryParse(static_cast<osmium::Relation &>(entity)));
                    break;
                default:
                    ++number_of_others;
                    break;
                }
            }
        }
        TIMER_STOP(parsing);
        SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
        SimpleLogger().Write() << "Raw input contains " << number_of_nodes << " nodes, " << number_of_ways << " ways, and " << number_of_relations << " relations";

        extractor_callbacks.reset();

        if (extraction_containers.all_edges_list.empty())
        {
            SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
            return 1;
        }

        extraction_containers.PrepareData(output_file_name, restriction_file_name);

        TIMER_STOP(extracting);
        SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
        SimpleLogger().Write() << "To prepare the data for routing, run: "
                               << "./osrm-prepare " << output_file_name << std::endl;
    }
    catch (boost::program_options::too_many_positional_options_error &)
    {
        SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
        return 1;
    }
    catch (std::exception &e)
    {
        SimpleLogger().Write(logWARNING) << e.what();
        return 1;
    }
    return 0;
}
<commit_msg>move LUA error callback function into anonymous namespace<commit_after>/*

Copyright (c) 2013, Project OSRM, Dennis Luxen, others
All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list
of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

*/

#include "Extractor.h"

#include "ExtractionContainers.h"
#include "ExtractionNode.h"
#include "ExtractionWay.h"
#include "ExtractorCallbacks.h"
#include "RestrictionParser.h"
#include "ScriptingEnvironment.h"

#include "../Util/GitDescription.h"
#include "../Util/IniFileUtil.h"
#include "../Util/OSRMException.h"
#include "../Util/simple_logger.hpp"
#include "../Util/TimingUtil.h"
#include "../Util/make_unique.hpp"
#include "../typedefs.h"

#include <boost/program_options.hpp>

#include <luabind/luabind.hpp>

#include <tbb/task_scheduler_init.h>

#include <osmium/io/any_input.hpp>

#include <cstdlib>

#include <chrono>
#include <fstream>
#include <iostream>
#include <thread>
#include <unordered_map>

Extractor::Extractor() : requested_num_threads(0), file_has_pbf_format(false)
{
}

namespace {
int lua_error_callback(lua_State *L) // This is so I can use my own function as an
// exception handler, pcall_log()
{
    luabind::object error_msg(luabind::from_stack(L, -1));
    std::ostringstream error_stream;
    error_stream << error_msg;
    throw OSRMException("ERROR occured in profile script:\n" + error_stream.str());
}
}

bool Extractor::ParseArguments(int argc, char *argv[])
{
    // declare a group of options that will be allowed only on command line
    boost::program_options::options_description generic_options("Options");
    generic_options.add_options()("version,v", "Show version")("help,h", "Show this help message")(
        "config,c",
        boost::program_options::value<boost::filesystem::path>(&config_file_path)
            ->default_value("extractor.ini"),
        "Path to a configuration file.");

    // declare a group of options that will be allowed both on command line and in config file
    boost::program_options::options_description config_options("Configuration");
    config_options.add_options()("profile,p",
                                 boost::program_options::value<boost::filesystem::path>(
                                     &profile_path)->default_value("profile.lua"),
                                 "Path to LUA routing profile")(
        "threads,t",
        boost::program_options::value<unsigned int>(&requested_num_threads)
            ->default_value(tbb::task_scheduler_init::default_num_threads()),
        "Number of threads to use");

    // hidden options, will be allowed both on command line and in config file, but will not be
    // shown to the user
    boost::program_options::options_description hidden_options("Hidden options");
    hidden_options.add_options()(
        "input,i",
        boost::program_options::value<boost::filesystem::path>(&input_path),
        "Input file in .osm, .osm.bz2 or .osm.pbf format");

    // positional option
    boost::program_options::positional_options_description positional_options;
    positional_options.add("input", 1);

    // combine above options for parsing
    boost::program_options::options_description cmdline_options;
    cmdline_options.add(generic_options).add(config_options).add(hidden_options);

    boost::program_options::options_description config_file_options;
    config_file_options.add(config_options).add(hidden_options);

    boost::program_options::options_description visible_options(
        boost::filesystem::basename(argv[0]) + " <input.osm/.osm.bz2/.osm.pbf> [options]");
    visible_options.add(generic_options).add(config_options);

    // parse command line options
    boost::program_options::variables_map option_variables;
    boost::program_options::store(boost::program_options::command_line_parser(argc, argv)
                                      .options(cmdline_options)
                                      .positional(positional_options)
                                      .run(),
                                  option_variables);

    if (option_variables.count("version"))
    {
        SimpleLogger().Write() << g_GIT_DESCRIPTION;
        return false;
    }

    if (option_variables.count("help"))
    {
        SimpleLogger().Write() << visible_options;
        return false;
    }

    boost::program_options::notify(option_variables);

    // parse config file
    if (boost::filesystem::is_regular_file(config_file_path))
    {
        SimpleLogger().Write() << "Reading options from: " << config_file_path.string();
        std::string ini_file_contents = ReadIniFileAndLowerContents(config_file_path);
        std::stringstream config_stream(ini_file_contents);
        boost::program_options::store(parse_config_file(config_stream, config_file_options),
                                      option_variables);
        boost::program_options::notify(option_variables);
    }

    if (!option_variables.count("input"))
    {
        SimpleLogger().Write() << visible_options;
        return false;
    }

    return true;
}

void Extractor::GenerateOutputFilesNames()
{
    output_file_name = input_path.string();
    restriction_file_name = input_path.string();
    timestamp_file_name = input_path.string();
    std::string::size_type pos = output_file_name.find(".osm.bz2");
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".osm.pbf");
        if (pos != std::string::npos)
        {
            file_has_pbf_format = true;
        }
        else
        {
            pos = output_file_name.find(".osm.xml");
        }
    }
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".pbf");
        if (pos != std::string::npos)
        {
            file_has_pbf_format = true;
        }
    }
    if (pos == std::string::npos)
    {
        pos = output_file_name.find(".osm");
        if (pos == std::string::npos)
        {
            output_file_name.append(".osrm");
            restriction_file_name.append(".osrm.restrictions");
            timestamp_file_name.append(".osrm.timestamp");
        }
        else
        {
            output_file_name.replace(pos, 5, ".osrm");
            restriction_file_name.replace(pos, 5, ".osrm.restrictions");
            timestamp_file_name.replace(pos, 5, ".osrm.timestamp");
        }
    }
    else
    {
        output_file_name.replace(pos, 8, ".osrm");
        restriction_file_name.replace(pos, 8, ".osrm.restrictions");
        timestamp_file_name.replace(pos, 8, ".osrm.timestamp");
    }
}

int Extractor::Run(int argc, char *argv[])
{
    try
    {
        LogPolicy::GetInstance().Unmute();

        TIMER_START(extracting);

        if (!ParseArguments(argc, argv))
            return 0;

        if (1 > requested_num_threads)
        {
            SimpleLogger().Write(logWARNING) << "Number of threads must be 1 or larger";
            return 1;
        }

        if (!boost::filesystem::is_regular_file(input_path))
        {
            SimpleLogger().Write(logWARNING) << "Input file " << input_path.string()
                                             << " not found!";
            return 1;
        }

        if (!boost::filesystem::is_regular_file(profile_path))
        {
            SimpleLogger().Write(logWARNING) << "Profile " << profile_path.string()
                                             << " not found!";
            return 1;
        }

        const unsigned recommended_num_threads = tbb::task_scheduler_init::default_num_threads();

        SimpleLogger().Write() << "Input file: " << input_path.filename().string();
        SimpleLogger().Write() << "Profile: " << profile_path.filename().string();
        SimpleLogger().Write() << "Threads: " << requested_num_threads;
        if (recommended_num_threads != requested_num_threads)
        {
            SimpleLogger().Write(logWARNING) << "The recommended number of threads is "
                                             << recommended_num_threads
                                             << "! This setting may have performance side-effects.";
        }

        tbb::task_scheduler_init init(requested_num_threads);

        /*** Setup Scripting Environment ***/
        ScriptingEnvironment scripting_environment(profile_path.string().c_str());

        GenerateOutputFilesNames();

        std::unordered_map<std::string, NodeID> string_map;
        ExtractionContainers extraction_containers;

        string_map[""] = 0;
        auto extractor_callbacks = osrm::make_unique<ExtractorCallbacks>(extraction_containers, string_map);

        osmium::io::File infile(input_path.string());
        osmium::io::Reader reader(infile);
        osmium::io::Header header = reader.header();

        unsigned number_of_nodes = 0;
        unsigned number_of_ways = 0;
        unsigned number_of_relations = 0;
        unsigned number_of_others = 0;

        SimpleLogger().Write() << "Parsing in progress..";
        TIMER_START(parsing);

        std::string generator = header.get("generator");
        if (generator.empty())
        {
            generator = "unknown tool";
        }
        SimpleLogger().Write() << "input file generated by " << generator;

        // TODO: write timestamp if non-empty
        std::string timestamp = header.get("osmosis_replication_timestamp");
        if (timestamp.empty())
        {
            timestamp = "n/a";
        }
        SimpleLogger().Write() << "timestamp: " << timestamp;

        boost::filesystem::ofstream timestamp_out(timestamp_file_name);
        timestamp_out.write(timestamp.c_str(), timestamp.length());
        timestamp_out.close();

        lua_State *lua_state = scripting_environment.getLuaState();
        luabind::set_pcall_callback(&lua_error_callback);

        RestrictionParser restriction_parser(scripting_environment);

        ExtractionNode result_node;
        ExtractionWay result_way;

        while (osmium::memory::Buffer buffer = reader.read())
        {
            for (osmium::OSMEntity &entity : buffer)
            {
                switch (entity.type())
                {
                case osmium::item_type::node:
                    ++number_of_nodes;
                    result_node.Clear();
                    luabind::call_function<void>(
                        lua_state,
                        "node_function",
                        boost::cref(static_cast<osmium::Node &>(entity)),
                        boost::ref(result_node));
                    extractor_callbacks->ProcessNode(static_cast<osmium::Node &>(entity),
                                                     result_node);
                    break;
                case osmium::item_type::way:
                    ++number_of_ways;
                    result_way.Clear();
                    luabind::call_function<void>(
                        lua_state,
                        "way_function",
                        boost::cref(static_cast<osmium::Way &>(entity)),
                        boost::ref(result_way));
                    extractor_callbacks->ProcessWay(static_cast<osmium::Way &>(entity),
                                                    result_way);
                    break;
                case osmium::item_type::relation:
                    ++number_of_relations;
                    extractor_callbacks->ProcessRestriction(
                        restriction_parser.TryParse(static_cast<osmium::Relation &>(entity)));
                    break;
                default:
                    ++number_of_others;
                    break;
                }
            }
        }
        TIMER_STOP(parsing);
        SimpleLogger().Write() << "Parsing finished after " << TIMER_SEC(parsing) << " seconds";
        SimpleLogger().Write() << "Raw input contains " << number_of_nodes << " nodes, " << number_of_ways << " ways, and " << number_of_relations << " relations";

        extractor_callbacks.reset();

        if (extraction_containers.all_edges_list.empty())
        {
            SimpleLogger().Write(logWARNING) << "The input data is empty, exiting.";
            return 1;
        }

        extraction_containers.PrepareData(output_file_name, restriction_file_name);

        TIMER_STOP(extracting);
        SimpleLogger().Write() << "extraction finished after " << TIMER_SEC(extracting) << "s";
        SimpleLogger().Write() << "To prepare the data for routing, run: "
                               << "./osrm-prepare " << output_file_name << std::endl;
    }
    catch (boost::program_options::too_many_positional_options_error &)
    {
        SimpleLogger().Write(logWARNING) << "Only one input file can be specified";
        return 1;
    }
    catch (std::exception &e)
    {
        SimpleLogger().Write(logWARNING) << e.what();
        return 1;
    }
    return 0;
}
<|endoftext|>