{"action":"created","comment":{"url":"https://api.github.com/repos/youtube/vitess/pulls/comments/60495826","id":60495826,"diff_hunk":"@@ -123,6 +154,153 @@ def load_schema(self):\n           if shard.keyspace == keyspace:\n             self.mysql_execute(cmds, db_name=shard.db_name)\n \n+  def populate_with_random_data(self):\n+    \"\"\"Populates all shards with randomly generated data.\"\"\"\n+\n+    for shard in self.shards:\n+      self.populate_shard_with_random_data(shard.db_name)\n+\n+  def populate_shard_with_random_data(self, db_name):\n+    \"\"\"Populates the given database with randomly generated data.\n+\n+    Every table in the database is populated.\n+\n+    Args:\n+      db_name: The shard database name (string).\n+    \"\"\"\n+\n+    tables = self.mysql_execute(['SHOW TABLES'], db_name)\n+    for table in tables:\n+      self.populate_table_with_random_data(db_name, table[0])\n+\n+  # The number of rows inserted in a single INSERT statement.\n+  batch_insert_size = 1000\n+\n+  def populate_table_with_random_data(self, db_name, table_name):\n+    \"\"\"Populates the given table with randomly generated data.\n+\n+    Queries the database for the table schema and then populates\n+    the columns with randomly generated data.\n+\n+    Args:\n+      db_name: The shard database name (string).\n+      table_name: The name of the table to populate (string).\n+    \"\"\"\n+\n+    field_infos = self.mysql_execute(['DESCRIBE %s' % table_name], db_name)\n+    num_rows = self.rng.randint(self.min_table_shard_size,\n+                                self.max_table_shard_size)\n+    rows = []\n+    for _ in xrange(num_rows):\n+      row = []\n+      for field_info in field_infos:\n+        field_type = field_info[1]\n+        field_allow_nulls = (field_info[2] == 'YES')\n+        row.append(\n+            self.generate_random_field(\n+                table_name, field_type, field_allow_nulls))\n+      rows.append(row)\n+\n+    # Insert 'rows' into the database in batches of size\n+    # self.batch_insert_size\n+    field_names = [field_info[0] for field_info in field_infos]\n+    for index in xrange(0, len(rows), self.batch_insert_size):\n+      self.batch_insert(db_name,\n+                        table_name,\n+                        field_names,\n+                        rows[index:index + self.batch_insert_size])\n+\n+  def batch_insert(self, db_name, table_name, field_names, rows):\n+    \"\"\"Inserts the rows in 'rows' into 'table_name' of database 'db_name'.\n+\n+    Args:\n+      db_name: The name of the database containing the table.\n+      table_name: The name of the table to populate.\n+      field_names: The list of the field names in the table.\n+      rows: A list of tuples with each tuple containing\n+          the string representations of the fields.\n+          The order of the representation must match the order of the field\n+          names listed in 'field_names'.\n+    \"\"\"\n+\n+    field_names_string = ','.join(field_names)\n+    values_string = ','.join(['(' + ','.join(row) +')' for row in rows])\n+    # We use \"INSERT IGNORE\" to ignore duplicate key errors.\n+    insert_query = ('INSERT IGNORE INTO %s (%s) VALUES %s' %\n+                    (table_name, field_names_string, values_string))\n+    self.mysql_execute([insert_query], db_name)\n+\n+  def generate_random_field(self, table_name, field_type, field_allows_nulls):\n+    \"\"\"Generates a random field string representation.\n+\n+    By 'string representation' we mean a string that is suitable to be a part\n+    of an 'INSERT INTO' SQL statement.\n+\n+    Args:\n+      table_name: The name of the table that will contain the generated field\n+          value. Only used for a descriptive exception message in case of\n+          an error.\n+      field_type: The field_type as given by a \"DESCRIBE <table>\" SQL statement.\n+      field_allows_nulls: Should be 'true' if this field allows NULLS.\n+\n+    Returns:\n+      The random field.\n+\n+    Raises:\n+      Exception: If 'field_type' is not supported.\n+    \"\"\"\n+\n+    value = None\n+    if field_type.startswith('tinyint'):\n+      value = self.random_integer(field_type, 1)\n+    elif field_type.startswith('smallint'):\n+      value = self.random_integer(field_type, 2)\n+    elif field_type.startswith('mediumint'):\n+      value = self.random_integer(field_type, 3)\n+    elif field_type.startswith('int'):\n+      value = self.random_integer(field_type, 4)\n+    elif field_type.startswith('bigint'):\n+      value = self.random_integer(field_type, 8)\n+    elif field_type.startswith('float'):\n+      value = self.random_float()\n+    elif field_type.startswith('double'):\n+      value = self.random_double()\n+    else:\n+      raise Exception('Populating random data in field type: %s is not yet '\n+                      'supported. (table: %s)' % (field_type, table_name))\n+    if field_allows_nulls and self.true_with_probability(self.null_probability):\n+      return 'NULL'\n+    return value\n+\n+  def true_with_probability(self, true_probability):\n+    \"\"\"Returns a pseudo-random boolean.\n+\n+    Args:\n+      true_probability: The probability to use for returning 'true'.\n+    Returns:\n+      The value 'true' is with probability 'true_probability'.\n+    \"\"\"\n+\n+    return self.rng.uniform(0, 1) < true_probability\n+\n+  def random_integer(self, field_type, num_bytes):\n+    num_bits = 8*num_bytes\n+    if field_type.endswith('unsigned'):\n+      return '%d' % (self.rng.randint(0, 2**num_bits-1))\n+    return '%d' % (self.rng.randint(-2**(num_bits-1), 2**(num_bits-1)-1))\n+\n+  def random_float(self):\n+    # Interpret a random 32-bit string as a float.","path":"py/vttest/local_database.py","position":214,"original_position":214,"commit_id":"5d11887f455e23bd3b16e0315be3d3f0f9026faf","original_commit_id":"5d11887f455e23bd3b16e0315be3d3f0f9026faf","user":{"login":"alainjobart","id":4625714,"avatar_url":"https://avatars.githubusercontent.com/u/4625714?v=3","gravatar_id":"","url":"https://api.github.com/users/alainjobart","html_url":"https://github.com/alainjobart","followers_url":"https://api.github.com/users/alainjobart/followers","following_url":"https://api.github.com/users/alainjobart/following{/other_user}","gists_url":"https://api.github.com/users/alainjobart/gists{/gist_id}","starred_url":"https://api.github.com/users/alainjobart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alainjobart/subscriptions","organizations_url":"https://api.github.com/users/alainjobart/orgs","repos_url":"https://api.github.com/users/alainjobart/repos","events_url":"https://api.github.com/users/alainjobart/events{/privacy}","received_events_url":"https://api.github.com/users/alainjobart/received_events","type":"User","site_admin":false},"body":"I think there are illegal values in there, like NAN... Maybe needs to be double-checked.","created_at":"2016-04-20T22:03:39Z","updated_at":"2016-04-20T22:03:39Z","html_url":"https://github.com/youtube/vitess/pull/1646#discussion_r60495826","pull_request_url":"https://api.github.com/repos/youtube/vitess/pulls/1646","_links":{"self":{"href":"https://api.github.com/repos/youtube/vitess/pulls/comments/60495826"},"html":{"href":"https://github.com/youtube/vitess/pull/1646#discussion_r60495826"},"pull_request":{"href":"https://api.github.com/repos/youtube/vitess/pulls/1646"}}},"pull_request":{"url":"https://api.github.com/repos/youtube/vitess/pulls/1646","id":67256505,"html_url":"https://github.com/youtube/vitess/pull/1646","diff_url":"https://github.com/youtube/vitess/pull/1646.diff","patch_url":"https://github.com/youtube/vitess/pull/1646.patch","issue_url":"https://api.github.com/repos/youtube/vitess/issues/1646","number":1646,"state":"open","locked":false,"title":"Adding a flag to run_local_database.py to allow populating the databa with initial random data.","user":{"login":"erzel","id":14099708,"avatar_url":"https://avatars.githubusercontent.com/u/14099708?v=3","gravatar_id":"","url":"https://api.github.com/users/erzel","html_url":"https://github.com/erzel","followers_url":"https://api.github.com/users/erzel/followers","following_url":"https://api.github.com/users/erzel/following{/other_user}","gists_url":"https://api.github.com/users/erzel/gists{/gist_id}","starred_url":"https://api.github.com/users/erzel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erzel/subscriptions","organizations_url":"https://api.github.com/users/erzel/orgs","repos_url":"https://api.github.com/users/erzel/repos","events_url":"https://api.github.com/users/erzel/events{/privacy}","received_events_url":"https://api.github.com/users/erzel/received_events","type":"User","site_admin":false},"body":"@alainjobart \r\n@dumbunny \r\nPlease take a look.\n\n<!-- Reviewable:start -->\n---\nThis change is [<img src=\"https://reviewable.io/review_button.svg\" height=\"35\" align=\"absmiddle\" alt=\"Reviewable\"/>](https://reviewable.io/reviews/youtube/vitess/1646)\n<!-- Reviewable:end -->\n","created_at":"2016-04-20T20:53:35Z","updated_at":"2016-04-20T22:03:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"46a427112f59d8184dfb45703b9e83104cf75ca5","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/youtube/vitess/pulls/1646/commits","review_comments_url":"https://api.github.com/repos/youtube/vitess/pulls/1646/comments","review_comment_url":"https://api.github.com/repos/youtube/vitess/pulls/comments{/number}","comments_url":"https://api.github.com/repos/youtube/vitess/issues/1646/comments","statuses_url":"https://api.github.com/repos/youtube/vitess/statuses/5d11887f455e23bd3b16e0315be3d3f0f9026faf","head":{"label":"erzel:add_gen_load_to_vttest","ref":"add_gen_load_to_vttest","sha":"5d11887f455e23bd3b16e0315be3d3f0f9026faf","user":{"login":"erzel","id":14099708,"avatar_url":"https://avatars.githubusercontent.com/u/14099708?v=3","gravatar_id":"","url":"https://api.github.com/users/erzel","html_url":"https://github.com/erzel","followers_url":"https://api.github.com/users/erzel/followers","following_url":"https://api.github.com/users/erzel/following{/other_user}","gists_url":"https://api.github.com/users/erzel/gists{/gist_id}","starred_url":"https://api.github.com/users/erzel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erzel/subscriptions","organizations_url":"https://api.github.com/users/erzel/orgs","repos_url":"https://api.github.com/users/erzel/repos","events_url":"https://api.github.com/users/erzel/events{/privacy}","received_events_url":"https://api.github.com/users/erzel/received_events","type":"User","site_admin":false},"repo":{"id":41829214,"name":"vitess","full_name":"erzel/vitess","owner":{"login":"erzel","id":14099708,"avatar_url":"https://avatars.githubusercontent.com/u/14099708?v=3","gravatar_id":"","url":"https://api.github.com/users/erzel","html_url":"https://github.com/erzel","followers_url":"https://api.github.com/users/erzel/followers","following_url":"https://api.github.com/users/erzel/following{/other_user}","gists_url":"https://api.github.com/users/erzel/gists{/gist_id}","starred_url":"https://api.github.com/users/erzel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erzel/subscriptions","organizations_url":"https://api.github.com/users/erzel/orgs","repos_url":"https://api.github.com/users/erzel/repos","events_url":"https://api.github.com/users/erzel/events{/privacy}","received_events_url":"https://api.github.com/users/erzel/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/erzel/vitess","description":"vitess provides servers and tools which facilitate scaling of MySQL databases for large scale web services.","fork":true,"url":"https://api.github.com/repos/erzel/vitess","forks_url":"https://api.github.com/repos/erzel/vitess/forks","keys_url":"https://api.github.com/repos/erzel/vitess/keys{/key_id}","collaborators_url":"https://api.github.com/repos/erzel/vitess/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/erzel/vitess/teams","hooks_url":"https://api.github.com/repos/erzel/vitess/hooks","issue_events_url":"https://api.github.com/repos/erzel/vitess/issues/events{/number}","events_url":"https://api.github.com/repos/erzel/vitess/events","assignees_url":"https://api.github.com/repos/erzel/vitess/assignees{/user}","branches_url":"https://api.github.com/repos/erzel/vitess/branches{/branch}","tags_url":"https://api.github.com/repos/erzel/vitess/tags","blobs_url":"https://api.github.com/repos/erzel/vitess/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/erzel/vitess/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/erzel/vitess/git/refs{/sha}","trees_url":"https://api.github.com/repos/erzel/vitess/git/trees{/sha}","statuses_url":"https://api.github.com/repos/erzel/vitess/statuses/{sha}","languages_url":"https://api.github.com/repos/erzel/vitess/languages","stargazers_url":"https://api.github.com/repos/erzel/vitess/stargazers","contributors_url":"https://api.github.com/repos/erzel/vitess/contributors","subscribers_url":"https://api.github.com/repos/erzel/vitess/subscribers","subscription_url":"https://api.github.com/repos/erzel/vitess/subscription","commits_url":"https://api.github.com/repos/erzel/vitess/commits{/sha}","git_commits_url":"https://api.github.com/repos/erzel/vitess/git/commits{/sha}","comments_url":"https://api.github.com/repos/erzel/vitess/comments{/number}","issue_comment_url":"https://api.github.com/repos/erzel/vitess/issues/comments{/number}","contents_url":"https://api.github.com/repos/erzel/vitess/contents/{+path}","compare_url":"https://api.github.com/repos/erzel/vitess/compare/{base}...{head}","merges_url":"https://api.github.com/repos/erzel/vitess/merges","archive_url":"https://api.github.com/repos/erzel/vitess/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/erzel/vitess/downloads","issues_url":"https://api.github.com/repos/erzel/vitess/issues{/number}","pulls_url":"https://api.github.com/repos/erzel/vitess/pulls{/number}","milestones_url":"https://api.github.com/repos/erzel/vitess/milestones{/number}","notifications_url":"https://api.github.com/repos/erzel/vitess/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/erzel/vitess/labels{/name}","releases_url":"https://api.github.com/repos/erzel/vitess/releases{/id}","deployments_url":"https://api.github.com/repos/erzel/vitess/deployments","created_at":"2015-09-02T21:42:27Z","updated_at":"2016-02-09T23:24:02Z","pushed_at":"2016-04-20T20:51:30Z","git_url":"git://github.com/erzel/vitess.git","ssh_url":"git@github.com:erzel/vitess.git","clone_url":"https://github.com/erzel/vitess.git","svn_url":"https://github.com/erzel/vitess","homepage":null,"size":58359,"stargazers_count":1,"watchers_count":1,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"youtube:master","ref":"master","sha":"daa1eddbb361297e64d138238f7a3d1c914f25de","user":{"login":"youtube","id":4052902,"avatar_url":"https://avatars.githubusercontent.com/u/4052902?v=3","gravatar_id":"","url":"https://api.github.com/users/youtube","html_url":"https://github.com/youtube","followers_url":"https://api.github.com/users/youtube/followers","following_url":"https://api.github.com/users/youtube/following{/other_user}","gists_url":"https://api.github.com/users/youtube/gists{/gist_id}","starred_url":"https://api.github.com/users/youtube/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/youtube/subscriptions","organizations_url":"https://api.github.com/users/youtube/orgs","repos_url":"https://api.github.com/users/youtube/repos","events_url":"https://api.github.com/users/youtube/events{/privacy}","received_events_url":"https://api.github.com/users/youtube/received_events","type":"Organization","site_admin":false},"repo":{"id":11008207,"name":"vitess","full_name":"youtube/vitess","owner":{"login":"youtube","id":4052902,"avatar_url":"https://avatars.githubusercontent.com/u/4052902?v=3","gravatar_id":"","url":"https://api.github.com/users/youtube","html_url":"https://github.com/youtube","followers_url":"https://api.github.com/users/youtube/followers","following_url":"https://api.github.com/users/youtube/following{/other_user}","gists_url":"https://api.github.com/users/youtube/gists{/gist_id}","starred_url":"https://api.github.com/users/youtube/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/youtube/subscriptions","organizations_url":"https://api.github.com/users/youtube/orgs","repos_url":"https://api.github.com/users/youtube/repos","events_url":"https://api.github.com/users/youtube/events{/privacy}","received_events_url":"https://api.github.com/users/youtube/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/youtube/vitess","description":"vitess provides servers and tools which facilitate scaling of MySQL databases for large scale web services.","fork":false,"url":"https://api.github.com/repos/youtube/vitess","forks_url":"https://api.github.com/repos/youtube/vitess/forks","keys_url":"https://api.github.com/repos/youtube/vitess/keys{/key_id}","collaborators_url":"https://api.github.com/repos/youtube/vitess/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/youtube/vitess/teams","hooks_url":"https://api.github.com/repos/youtube/vitess/hooks","issue_events_url":"https://api.github.com/repos/youtube/vitess/issues/events{/number}","events_url":"https://api.github.com/repos/youtube/vitess/events","assignees_url":"https://api.github.com/repos/youtube/vitess/assignees{/user}","branches_url":"https://api.github.com/repos/youtube/vitess/branches{/branch}","tags_url":"https://api.github.com/repos/youtube/vitess/tags","blobs_url":"https://api.github.com/repos/youtube/vitess/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/youtube/vitess/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/youtube/vitess/git/refs{/sha}","trees_url":"https://api.github.com/repos/youtube/vitess/git/trees{/sha}","statuses_url":"https://api.github.com/repos/youtube/vitess/statuses/{sha}","languages_url":"https://api.github.com/repos/youtube/vitess/languages","stargazers_url":"https://api.github.com/repos/youtube/vitess/stargazers","contributors_url":"https://api.github.com/repos/youtube/vitess/contributors","subscribers_url":"https://api.github.com/repos/youtube/vitess/subscribers","subscription_url":"https://api.github.com/repos/youtube/vitess/subscription","commits_url":"https://api.github.com/repos/youtube/vitess/commits{/sha}","git_commits_url":"https://api.github.com/repos/youtube/vitess/git/commits{/sha}","comments_url":"https://api.github.com/repos/youtube/vitess/comments{/number}","issue_comment_url":"https://api.github.com/repos/youtube/vitess/issues/comments{/number}","contents_url":"https://api.github.com/repos/youtube/vitess/contents/{+path}","compare_url":"https://api.github.com/repos/youtube/vitess/compare/{base}...{head}","merges_url":"https://api.github.com/repos/youtube/vitess/merges","archive_url":"https://api.github.com/repos/youtube/vitess/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/youtube/vitess/downloads","issues_url":"https://api.github.com/repos/youtube/vitess/issues{/number}","pulls_url":"https://api.github.com/repos/youtube/vitess/pulls{/number}","milestones_url":"https://api.github.com/repos/youtube/vitess/milestones{/number}","notifications_url":"https://api.github.com/repos/youtube/vitess/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/youtube/vitess/labels{/name}","releases_url":"https://api.github.com/repos/youtube/vitess/releases{/id}","deployments_url":"https://api.github.com/repos/youtube/vitess/deployments","created_at":"2013-06-27T21:20:28Z","updated_at":"2016-04-20T22:02:51Z","pushed_at":"2016-04-20T21:38:38Z","git_url":"git://github.com/youtube/vitess.git","ssh_url":"git@github.com:youtube/vitess.git","clone_url":"https://github.com/youtube/vitess.git","svn_url":"https://github.com/youtube/vitess","homepage":null,"size":83681,"stargazers_count":3407,"watchers_count":3407,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":430,"mirror_url":null,"open_issues_count":14,"forks":430,"open_issues":14,"watchers":3407,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/youtube/vitess/pulls/1646"},"html":{"href":"https://github.com/youtube/vitess/pull/1646"},"issue":{"href":"https://api.github.com/repos/youtube/vitess/issues/1646"},"comments":{"href":"https://api.github.com/repos/youtube/vitess/issues/1646/comments"},"review_comments":{"href":"https://api.github.com/repos/youtube/vitess/pulls/1646/comments"},"review_comment":{"href":"https://api.github.com/repos/youtube/vitess/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/youtube/vitess/pulls/1646/commits"},"statuses":{"href":"https://api.github.com/repos/youtube/vitess/statuses/5d11887f455e23bd3b16e0315be3d3f0f9026faf"}}}}