ran full testing with updated prompt, improves team and other_stats performance, slight drop on game performance

test_pretrained.ipynb

@@ -16,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -26,9 +26,9 @@
       "Total dataset examples: 1044\n",
       "\n",
       "\n",
-      "What was the largest deficit overcome by the Miami Heat in any home victory?\n",
-      "SELECT o.largest_lead_away AS max_deficit_overcome FROM game g JOIN other_stats o ON g.game_id = o.game_id WHERE g.team_name_home = 'Miami Heat' AND g.wl_home = 'W' ORDER BY o.largest_lead_away DESC LIMIT 1;\n",
-      "46\n"
+      "Which team committed the fewest total turnovers in an away game that resulted in a win?\n",
+      "SELECT team_abbreviation_away FROM other_stats WHERE game_id IN (SELECT game_id FROM game WHERE wl_away = 'W') ORDER BY total_turnovers_away ASC LIMIT 1;\n",
+      "PHX\n"
      ]
     }
    ],
@@ -58,7 +58,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -236,21 +236,8 @@
     "\n",
     "Example: To get statistics from 2005, use a statement like: season_id = '22005'. To get statistics from 1972, use a statement like: season_id = \"21972\". To get statistics from 2015, use a statement like: season_id = \"22015\".\n",
     "\n",
-    "The game_id column can be used to join the game and other_stats tables.\n",
-    "\n",
     "Ensure queries return relevant columns and avoid unnecessary joins.\n",
     "\n",
-    "When obtaining certain statistics by team from the game table, use the team_name_home and team_name_away columns. \n",
-    "For example, to obtain home game data for the Washington Wizards from the game table use a statement like: team_name_home = 'Washington Wizards'\n",
-    "To obtain away game data from the Los Angeles Lakers from the game table use a statement like: team_name_away = 'Los Angeles Lakers'\n",
-    "To obtain general game data where home or away is not specified for the Chicago Bulls from the game table, use a statement like: (team_name_home = 'Chicago Bulls' OR team_name_away = 'Chicago Bulls')\n",
-    "\n",
-    "When obtaining certain statistics by team from the other_stats table, use the team_abbreviation_home and team_abbreviation away columns.\n",
-    "For example, to obtain home statistics from the Charlotte Hornets from the other_stats table use a statement like: team_abbreviation_home = 'CHA'\n",
-    "To obtain away statistics from the Dallas Mavericks from the other_stats table, use a statement like: team_abbreviation_away = 'DAL'\n",
-    "To obtain general statistics from the other_stats table where home or away is not specified for the Detroit Pistons use a statement like: (team_abbreviation_home = 'DET' OR team_abbreviation_away = 'DET)\n",
-    "\n",
-    "\n",
     "Example User Requests and SQLite Queries\n",
     "Request:\n",
     "\"What is the most points the Los Angeles Lakers have ever scored at home?\"\n",
@@ -265,62 +252,22 @@
     "SELECT full_name FROM team WHERE state = 'California';\n",
     "\n",
     "Request:\n",
-    "\"How many total team rebounds did the Los Angeles Clippers have in away games where they scored over 15 fast break points?\"\n",
-    "SQLite:\n",
-    "SELECT SUM(os.team_rebounds_away) \n",
-    "FROM other_stats os  \n",
-    "JOIN game g ON os.game_id = g.game_id  \n",
-    "WHERE g.team_abbreviation_away = 'LAC' AND os.pts_fb_away > 15;\n",
-    "\n",
-    "Request:\n",
-    "\"How many points did the Miami Heat score on January 10, 2010?\"\n",
-    "SQLite:\n",
-    "SELECT team_name_home, pts_home, team_name_away, pts_away \n",
-    "FROM game \n",
-    "WHERE DATE(game_date) = '2010-01-10' \n",
-    "AND (team_name_home = 'Miami Heat' OR team_name_away = 'Miami Heat');\n",
-    "\n",
-    "Request:\n",
     "\"Which team had the highest number of team turnovers in an away game?\"\n",
     "SQLite:\n",
     "SELECT team_abbreviation_away FROM other_stats ORDER BY team_turnovers_away DESC LIMIT 1;\n",
     "\n",
     "Request:\n",
-    "\"Which team won the most home games in the 2000 season?\"\n",
-    "SQLite:\n",
-    "SELECT team_name_home, COUNT(*) AS wins\n",
-    "FROM game\n",
-    "WHERE wl_home = 'W' AND season_id = '22000'\n",
-    "GROUP BY team_name_home\n",
-    "ORDER BY wins DESC\n",
-    "LIMIT 1;\n",
-    "\n",
-    "Request:\n",
     "\"Which teams were founded before 1979?\"\n",
     "SQLite:\n",
     "SELECT full_name FROM team WHERE year_founded < 1979;\n",
     "\n",
     "Request:\n",
-    "\"Which game had the most lead changes in the 2020 season?\"\n",
-    "SQLite:\n",
-    "SELECT game_id, lead_changes  \n",
-    "FROM other_stats \n",
-    "WHERE game_id IN \n",
-    "(SELECT game_id FROM game WHERE season_id = '22020')\n",
-    "ORDER BY lead_changes DESC LIMIT 1;\n",
-    "\n",
-    "Request:\n",
     "\"Find the Boston Celtics largest home victory margin in the 2008 season.\"\n",
     "SQLite:\n",
     "SELECT MAX(pts_home - pts_away) AS biggest_win\n",
     "FROM game\n",
     "WHERE team_name_home = 'Boston Celtics' AND season_id = '22008';\n",
     "\n",
-    "Request:\n",
-    "\"How many fast break points did the Atlanta Hawks score at home?\"\n",
-    "SQLite:\n",
-    "SELECT SUM(pts_fb_home) as total_fb_points  FROM other_stats  WHERE team_abbreviation_home = 'ATL';\n",
-    "\n",
     "Generate only the SQLite query prefaced by SQLite: and no other text, do not output an explanation of the query. Now generate an SQLite query for the following user request. Request:\n",
     "\"\"\""
    ]
@@ -334,7 +281,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -342,9 +289,9 @@
      "output_type": "stream",
      "text": [
       "SQLite:\n",
-      "SELECT MAX(pts_home - pts_away) AS largest_deficit\n",
-      "FROM game\n",
-      "WHERE wl_home = 'W' AND team_name_home = 'Miami Heat';\n",
+      "SELECT team_abbreviation_away \n",
+      "FROM other_stats \n",
+      "WHERE wl_away = 'W' AND total_turnovers_away < (SELECT MIN(total_turnovers_away) FROM other_stats WHERE wl_away = 'L');\n",
       "\n"
      ]
     }
@@ -369,15 +316,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "cleaned\n",
-      "(43.0,)\n"
+      "cleaned\n"
      ]
     }
    ],
@@ -415,24 +361,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "How many home games did the Chicago Bulls play in the 2020 season?\n",
-      "SELECT COUNT(*) FROM game WHERE team_name_home = 'Chicago Bulls' AND season_id = '22020';\n",
-      "36.0\n",
+      "What is the largest lead the Minnesota Timberwolves had at home?\n",
+      "SELECT MAX(largest_lead_home) as max_lead  FROM other_stats  WHERE team_abbreviation_home = 'MIN';\n",
+      "48.0\n",
       "SQLite:\n",
-      "SELECT COUNT(*) as total_home_games \n",
-      "FROM game \n",
-      "WHERE team_name_home = 'Chicago Bulls' AND season_id = '22020';\n",
+      "SELECT MAX(largest_lead_home) \n",
+      "FROM other_stats \n",
+      "WHERE team_name_home = 'Minnesota Timberwolves';\n",
       "\n",
-      "Statement valid? True\n",
+      "Statement valid? False\n",
       "SQLite matched? False\n",
-      "Result matched? True\n"
+      "Result matched? False\n"
      ]
     }
    ],
@@ -561,7 +507,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -609,7 +555,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -617,18 +563,21 @@
      "output_type": "stream",
      "text": [
       "Completed 50\n",
+      "Completed 100\n",
+      "Completed 150\n",
+      "Completed 200\n",
       "\n",
       "Less than 90 results:\n",
-      "Percent valid: 0.62\n",
-      "Percent SQLite matched: 0.12\n",
-      "Percent result matched: 0.4\n",
+      "Percent valid: 0.8448979591836735\n",
+      "Percent SQLite matched: 0.43673469387755104\n",
+      "Percent result matched: 0.6530612244897959\n",
       "Dataset length: 245\n"
      ]
     }
    ],
    "source": [
     "less_than_90_df = pd.read_csv(\"./train-data/less_than_90.tsv\", sep='\\t')\n",
-    "run_evaluation(less_than_90_df.sample(n=50), \"Less than 90\")\n",
+    "run_evaluation(less_than_90_df, \"Less than 90\")\n",
     "print(\"Dataset length: \" + str(len(less_than_90_df)))"
    ]
   },
@@ -641,7 +590,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -666,9 +615,9 @@
       "Completed 800\n",
       "\n",
       "Queries from game results:\n",
-      "Percent valid: 0.6181384248210023\n",
-      "Percent SQLite matched: 0.015513126491646777\n",
-      "Percent result matched: 0.24343675417661098\n",
+      "Percent valid: 0.7613365155131265\n",
+      "Percent SQLite matched: 0.13842482100238662\n",
+      "Percent result matched: 0.383054892601432\n",
       "Dataset length: 838\n"
      ]
     }
@@ -688,7 +637,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -700,9 +649,9 @@
       "Completed 150\n",
       "\n",
       "Queries from other stats results:\n",
-      "Percent valid: 0.6168831168831169\n",
-      "Percent SQLite matched: 0.06493506493506493\n",
-      "Percent result matched: 0.34415584415584416\n",
+      "Percent valid: 0.21428571428571427\n",
+      "Percent SQLite matched: 0.01948051948051948\n",
+      "Percent result matched: 0.07142857142857142\n",
       "Dataset length: 154\n"
      ]
     }
@@ -722,7 +671,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -732,9 +681,9 @@
       "Completed 50\n",
       "\n",
       "Queries from team results:\n",
-      "Percent valid: 0.8846153846153846\n",
-      "Percent SQLite matched: 0.6346153846153846\n",
-      "Percent result matched: 0.8269230769230769\n",
+      "Percent valid: 0.8653846153846154\n",
+      "Percent SQLite matched: 0.5961538461538461\n",
+      "Percent result matched: 0.7884615384615384\n",
       "Dataset length: 52\n"
      ]
     }
@@ -754,7 +703,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -766,9 +715,9 @@
       "Completed 150\n",
       "\n",
       "Queries with join results:\n",
-      "Percent valid: 0.06486486486486487\n",
+      "Percent valid: 0.1945945945945946\n",
       "Percent SQLite matched: 0.0\n",
-      "Percent result matched: 0.010810810810810811\n",
+      "Percent result matched: 0.04864864864864865\n",
       "Dataset length: 185\n"
      ]
     }
@@ -788,7 +737,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -814,9 +763,9 @@
       "Completed 850\n",
       "\n",
       "Queries without join results:\n",
-      "Percent valid: 0.7974388824214202\n",
-      "Percent SQLite matched: 0.1559953434225844\n",
-      "Percent result matched: 0.4318975552968568\n",
+      "Percent valid: 0.7916181606519208\n",
+      "Percent SQLite matched: 0.17462165308498254\n",
+      "Percent result matched: 0.42374854481955765\n",
       "Dataset length: 859\n"
      ]
     }
@@ -836,7 +785,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -865,8 +814,8 @@
       "Completed 1000\n",
       "\n",
       "All training data results:\n",
-      "Percent valid: 0.6676245210727969\n",
-      "Percent SQLite matched: 0.12835249042145594\n",
+      "Percent valid: 0.685823754789272\n",
+      "Percent SQLite matched: 0.14367816091954022\n",
       "Percent result matched: 0.35823754789272033\n",
       "Dataset length: 1044\n"
      ]