{"per_item": [{"predicted": "SELECT Count(*) FROM stadium", "gold": "SELECT count(*) FROM singer", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM stadium", "gold": "SELECT count(*) FROM singer", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT singer.Name, singer.Country, singer.Age FROM singer ORDER BY singer.Age Desc", "gold": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name, singer.Country, singer.Age FROM singer ORDER BY singer.Age Desc", "gold": "SELECT name ,  country ,  age FROM singer ORDER BY age DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Avg(singer.Age), Min(singer.Age), Max(singer.Age) FROM singer WHERE singer.Country = 'terminal'", "gold": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(singer.Age), Min(singer.Age), Max(singer.Age) FROM singer WHERE singer.Country = 'terminal'", "gold": "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name, singer.Song_release_year FROM singer ORDER BY singer.Age Asc LIMIT 1", "gold": "SELECT song_name ,  song_release_year FROM singer ORDER BY age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Name, singer.Song_release_year FROM singer ORDER BY singer.Age Asc LIMIT 1", "gold": "SELECT song_name ,  song_release_year FROM singer ORDER BY age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT DISTINCT singer.Country FROM singer WHERE singer.Age > 'terminal'", "gold": "SELECT DISTINCT country FROM singer WHERE age  >  20", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT singer.Country FROM singer WHERE singer.Age > 'terminal'", "gold": "SELECT DISTINCT country FROM singer WHERE age  >  20", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Country, Count(*) FROM singer GROUP BY singer.Country", "gold": "SELECT country ,  count(*) FROM singer GROUP BY country", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Country, Count(*) FROM singer GROUP BY singer.Country", "gold": "SELECT country ,  count(*) FROM singer GROUP BY country", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Song_Name FROM singer WHERE singer.Age > (SELECT Avg(singer.Age) FROM singer)", "gold": "SELECT song_name FROM singer WHERE age  >  (SELECT avg(age) FROM singer)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Age > (SELECT Avg(singer.Age) FROM singer)", "gold": "SELECT song_name FROM singer WHERE age  >  (SELECT avg(age) FROM singer)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Location, stadium.Name FROM stadium WHERE stadium.Capacity BETWEEN 'terminal' AND 'terminal'", "gold": "SELECT LOCATION ,  name FROM stadium WHERE capacity BETWEEN 5000 AND 10000", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Location, stadium.Name FROM stadium WHERE stadium.Capacity BETWEEN 'terminal' AND 'terminal'", "gold": "SELECT LOCATION ,  name FROM stadium WHERE capacity BETWEEN 5000 AND 10000", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Max(stadium.Capacity), Avg(stadium.Capacity) FROM stadium", "gold": "select max(capacity), average from stadium", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(stadium.Capacity), Max(stadium.Capacity) FROM stadium", "gold": "select avg(capacity) ,  max(capacity) from stadium", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT stadium.Name, stadium.Capacity FROM stadium GROUP BY stadium.Stadium_ID ORDER BY Avg(stadium.Average) Desc LIMIT 1", "gold": "SELECT name ,  capacity FROM stadium ORDER BY average DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT stadium.Name, stadium.Capacity FROM stadium GROUP BY stadium.Stadium_ID ORDER BY Avg(stadium.Average) Desc LIMIT 1", "gold": "SELECT name ,  capacity FROM stadium ORDER BY average DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT Count(*) FROM concert WHERE concert.Year = 'terminal' OR concert.Year = 'terminal'", "gold": "SELECT count(*) FROM concert WHERE YEAR  =  2014 OR YEAR  =  2015", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM concert WHERE concert.Year = 'terminal' OR concert.Year = 'terminal'", "gold": "SELECT count(*) FROM concert WHERE YEAR  =  2014 OR YEAR  =  2015", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT stadium.Name, Count(*) FROM stadium JOIN concert GROUP BY concert.Stadium_ID", "gold": "SELECT T2.name ,  count(*) FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id GROUP BY T1.stadium_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Stadium_ID, Count(*) FROM stadium JOIN concert GROUP BY stadium.Stadium_ID", "gold": "SELECT T2.name ,  count(*) FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id GROUP BY T1.stadium_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Name, stadium.Capacity FROM stadium JOIN concert WHERE stadium.Highest >= 'terminal' GROUP BY concert.Stadium_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.name ,  T2.capacity FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.year  >=  2014 GROUP BY T2.stadium_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 5, "pred_total": 5}}}, {"predicted": "SELECT stadium.Name, stadium.Capacity FROM stadium JOIN concert WHERE concert.Year > 'terminal' GROUP BY concert.Stadium_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "select t2.name ,  t2.capacity from concert as t1 join stadium as t2 on t1.stadium_id  =  t2.stadium_id where t1.year  >  2013 group by t2.stadium_id order by count(*) desc limit 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 5, "pred_total": 5}}}, {"predicted": "SELECT concert.Year FROM concert GROUP BY concert.Year ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT YEAR FROM concert GROUP BY YEAR ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT concert.Year FROM concert GROUP BY concert.Year ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT YEAR FROM concert GROUP BY YEAR ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT stadium.Name FROM stadium WHERE stadium.Stadium_ID NOT IN (SELECT concert.Stadium_ID FROM concert)", "gold": "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT stadium.Name FROM stadium WHERE stadium.Stadium_ID NOT IN (SELECT concert.Stadium_ID FROM concert)", "gold": "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Country FROM singer WHERE singer.Age > 'terminal' INTERSECT SELECT singer.Country FROM singer WHERE singer.Age < 'terminal'", "gold": "SELECT country FROM singer WHERE age  >  40 INTERSECT SELECT country FROM singer WHERE age  <  30", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT stadium.Name FROM stadium EXCEPT SELECT stadium.Name FROM stadium JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT name FROM stadium EXCEPT SELECT T2.name FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.year  =  2014", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Name FROM stadium EXCEPT SELECT stadium.Name FROM stadium JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT name FROM stadium EXCEPT SELECT T2.name FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.year  =  2014", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT concert.concert_Name, concert.Theme, Count(*) FROM singer_in_concert JOIN concert GROUP BY singer_in_concert.concert_ID", "gold": "SELECT T2.concert_name ,  T2.theme ,  count(*) FROM singer_in_concert AS T1 JOIN concert AS T2 ON T1.concert_id  =  T2.concert_id GROUP BY T2.concert_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT concert.Theme, concert.concert_Name, Count(*) FROM singer_in_concert JOIN concert GROUP BY singer_in_concert.concert_ID", "gold": "select t2.concert_name ,  t2.theme ,  count(*) from singer_in_concert as t1 join concert as t2 on t1.concert_id  =  t2.concert_id group by t2.concert_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name, Count(*) FROM singer_in_concert JOIN singer GROUP BY singer.Name", "gold": "SELECT T2.name ,  count(*) FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id GROUP BY T2.singer_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT concert.concert_Name, Count(*) FROM singer_in_concert JOIN singer GROUP BY singer.Name", "gold": "SELECT T2.name ,  count(*) FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id GROUP BY T2.singer_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT T2.name FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id JOIN concert AS T3 ON T1.concert_id  =  T3.concert_id WHERE T3.year  =  2014", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT T2.name FROM singer_in_concert AS T1 JOIN singer AS T2 ON T1.singer_id  =  T2.singer_id JOIN concert AS T3 ON T1.concert_id  =  T3.concert_id WHERE T3.year  =  2014", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name, singer.Country FROM singer WHERE singer.Song_Name LIKE 'terminal'", "gold": "SELECT name ,  country FROM singer WHERE song_name LIKE '%Hey%'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name, singer.Country FROM singer WHERE singer.Song_Name LIKE 'terminal'", "gold": "SELECT name ,  country FROM singer WHERE song_name LIKE '%Hey%'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT stadium.Name, stadium.Location FROM stadium JOIN concert WHERE concert.Year = 'terminal' INTERSECT SELECT stadium.Name, stadium.Location FROM stadium JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT T2.name ,  T2.location FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.Year  =  2014 INTERSECT SELECT T2.name ,  T2.location FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.Year  =  2015", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT stadium.Name, stadium.Location FROM stadium JOIN concert WHERE concert.Year = 'terminal' INTERSECT SELECT stadium.Name, stadium.Location FROM stadium JOIN concert WHERE concert.Year = 'terminal'", "gold": "SELECT T2.name ,  T2.location FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.Year  =  2014 INTERSECT SELECT T2.name ,  T2.location FROM concert AS T1 JOIN stadium AS T2 ON T1.stadium_id  =  T2.stadium_id WHERE T1.Year  =  2015", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM stadium WHERE stadium.Capacity = (SELECT Max(stadium.Capacity) FROM stadium)", "gold": "select count(*) from concert where stadium_id = (select stadium_id from stadium order by capacity desc limit 1)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT stadium.Capacity FROM stadium ORDER BY stadium.Capacity Desc LIMIT 1", "gold": "select count(*) from concert where stadium_id = (select stadium_id from stadium order by capacity desc limit 1)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Pets WHERE Pets.weight > 'terminal'", "gold": "SELECT count(*) FROM pets WHERE weight  >  10", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Pets WHERE Pets.weight > 'terminal'", "gold": "SELECT count(*) FROM pets WHERE weight  >  10", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Pets.weight FROM Has_Pet JOIN Pets JOIN Student WHERE Student.Age = (SELECT Min(Student.Age) FROM Student)", "gold": "SELECT weight FROM pets ORDER BY pet_age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Student.Age FROM Student WHERE Student.Age = (SELECT Min(Student.Age) FROM Student)", "gold": "SELECT weight FROM pets ORDER BY pet_age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Max(Pets.weight), Pets.PetType FROM Pets GROUP BY Pets.PetType", "gold": "SELECT max(weight) ,  petType FROM pets GROUP BY petType", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Max(Pets.weight), Min(Pets.weight), Pets.PetType FROM Pets GROUP BY Pets.PetType", "gold": "SELECT max(weight) ,  petType FROM pets GROUP BY petType", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Has_Pet JOIN Student JOIN Pets WHERE Student.Age > 'terminal'", "gold": "SELECT count(*) FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T1.age  >  20", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Has_Pet JOIN Student WHERE Student.Age > 'terminal'", "gold": "SELECT count(*) FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T1.age  >  20", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Student JOIN Has_Pet WHERE Student.Sex = 'terminal'", "gold": "SELECT count(*) FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T2.petid  =  T3.petid WHERE T1.sex  =  'F' AND T3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Has_Pet JOIN Student WHERE Student.Sex = 'terminal'", "gold": "SELECT count(*) FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T2.petid  =  T3.petid WHERE T1.sex  =  'F' AND T3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT Pets.PetType) FROM Pets", "gold": "SELECT count(DISTINCT pettype) FROM pets", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Pets.PetType) FROM Pets", "gold": "SELECT count(DISTINCT pettype) FROM pets", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Student.Fname FROM Student JOIN Has_Pet JOIN Pets WHERE Has_Pet.PetID = 'terminal' OR Pets.PetType = 'terminal'", "gold": "SELECT DISTINCT T1.Fname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat' OR T3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Student.Fname FROM Has_Pet JOIN Pets JOIN Student WHERE Student.city_code = 'terminal' OR Pets.PetType = 'terminal'", "gold": "SELECT DISTINCT T1.Fname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat' OR T3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Student.Fname FROM Has_Pet JOIN Student INTERSECT SELECT Student.Fname FROM Has_Pet JOIN Student", "gold": "select t1.fname from student as t1 join has_pet as t2 on t1.stuid  =  t2.stuid join pets as t3 on t3.petid  =  t2.petid where t3.pettype  =  'cat' intersect select t1.fname from student as t1 join has_pet as t2 on t1.stuid  =  t2.stuid join pets as t3 on t3.petid  =  t2.petid where t3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Student.Fname FROM Has_Pet JOIN Pets JOIN Student WHERE Pets.PetType = 'terminal' INTERSECT SELECT Student.Fname FROM Has_Pet JOIN Pets JOIN Student WHERE Pets.PetType = 'terminal'", "gold": "SELECT T1.Fname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat' INTERSECT SELECT T1.Fname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'dog'", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Student.Major, Student.Age FROM Student WHERE Student.StuID NOT IN (SELECT Has_Pet.StuID FROM Has_Pet)", "gold": "SELECT major ,  age FROM student WHERE stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Student.Major, Student.Age FROM Student WHERE Student.StuID NOT IN (SELECT Has_Pet.StuID FROM Has_Pet)", "gold": "SELECT major ,  age FROM student WHERE stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Student.StuID FROM Student EXCEPT SELECT Has_Pet.StuID FROM Has_Pet", "gold": "SELECT stuid FROM student EXCEPT SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Student.StuID FROM Student EXCEPT SELECT Has_Pet.StuID FROM Has_Pet", "gold": "SELECT stuid FROM student EXCEPT SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Student.Fname, Student.Age FROM Student WHERE Student.StuID NOT IN (SELECT Has_Pet.StuID FROM Has_Pet)", "gold": "SELECT T1.fname ,  T1.age FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'dog' AND T1.stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Student.Fname FROM Student WHERE Student.StuID IN (SELECT Has_Pet.StuID FROM Has_Pet) EXCEPT SELECT Student.Fname FROM Student JOIN Has_Pet JOIN Pets", "gold": "SELECT T1.fname ,  T1.age FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'dog' AND T1.stuid NOT IN (SELECT T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pettype  =  'cat')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Pets.PetType, Pets.weight FROM Has_Pet JOIN Pets JOIN Student WHERE Student.Age = (SELECT Max(Student.Age) FROM Student)", "gold": "SELECT pettype ,  weight FROM pets ORDER BY pet_age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Pets.PetType, Pets.weight FROM Has_Pet JOIN Pets JOIN Student ORDER BY Student.Age Asc LIMIT 1", "gold": "SELECT pettype ,  weight FROM pets ORDER BY pet_age LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Pets.PetID, Pets.weight FROM Has_Pet JOIN Student JOIN Pets WHERE Student.Age > 'terminal'", "gold": "SELECT petid ,  weight FROM pets WHERE pet_age  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Pets.PetID, Pets.weight FROM Pets WHERE Pets.pet_age > 'terminal'", "gold": "SELECT petid ,  weight FROM pets WHERE pet_age  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(Student.Age), Max(Student.Age), Max(Pets.weight) FROM Has_Pet JOIN Student JOIN Pets", "gold": "SELECT avg(pet_age) ,  max(pet_age) ,  pettype FROM pets GROUP BY pettype", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Avg(Student.Age), Max(Student.Age), Max(Student.Age) FROM Student", "gold": "SELECT avg(pet_age) ,  max(pet_age) ,  pettype FROM pets GROUP BY pettype", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Pets.PetType, Avg(Pets.weight) FROM Pets GROUP BY Pets.PetType", "gold": "SELECT avg(weight) ,  pettype FROM pets GROUP BY pettype", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(Pets.weight), Pets.PetType FROM Pets GROUP BY Pets.PetType", "gold": "SELECT avg(weight) ,  pettype FROM pets GROUP BY pettype", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Student.Fname, Student.Age FROM Student JOIN Has_Pet", "gold": "SELECT DISTINCT T1.fname ,  T1.age FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT Student.Fname, Student.Age FROM Has_Pet JOIN Student JOIN Pets", "gold": "SELECT DISTINCT T1.fname ,  T1.age FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Has_Pet.PetID FROM Has_Pet JOIN Student JOIN Pets WHERE Student.LName = 'terminal'", "gold": "SELECT T2.petid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T1.Lname  =  'Smith'", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Has_Pet.PetID FROM Has_Pet JOIN Student WHERE Student.LName = 'terminal'", "gold": "SELECT T2.petid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid WHERE T1.Lname  =  'Smith'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Has_Pet.StuID FROM Has_Pet GROUP BY Has_Pet.StuID", "gold": "SELECT count(*) ,  T1.stuid FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid GROUP BY T1.stuid", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Has_Pet.StuID FROM Has_Pet GROUP BY Has_Pet.StuID", "gold": "select count(*) ,  t1.stuid from student as t1 join has_pet as t2 on t1.stuid  =  t2.stuid group by t1.stuid", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Student.Fname, Student.Sex FROM Has_Pet JOIN Student GROUP BY Has_Pet.StuID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.fname ,  T1.sex FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid GROUP BY T1.stuid HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Student.Fname, Student.Sex FROM Has_Pet JOIN Student GROUP BY Has_Pet.StuID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.fname ,  T1.sex FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid GROUP BY T1.stuid HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Student.LName FROM Student WHERE Student.StuID IN (SELECT Has_Pet.StuID FROM Has_Pet WHERE Student.Age = 'terminal')", "gold": "SELECT T1.lname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pet_age  =  3 AND T3.pettype  =  'cat'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Student.LName FROM Student WHERE Student.StuID IN (SELECT Has_Pet.StuID FROM Has_Pet JOIN Pets WHERE Pets.pet_age = 'terminal')", "gold": "SELECT T1.lname FROM student AS T1 JOIN has_pet AS T2 ON T1.stuid  =  T2.stuid JOIN pets AS T3 ON T3.petid  =  T2.petid WHERE T3.pet_age  =  3 AND T3.pettype  =  'cat'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Avg(Student.Age) FROM Student WHERE Student.StuID NOT IN (SELECT Has_Pet.StuID FROM Has_Pet)", "gold": "select avg(age) from student where stuid not in (select stuid from has_pet)", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(Student.Age) FROM Student WHERE Student.StuID NOT IN (SELECT Has_Pet.StuID FROM Has_Pet)", "gold": "select avg(age) from student where stuid not in (select stuid from has_pet)", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM countries", "gold": "SELECT count(*) FROM CONTINENTS;", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM continents", "gold": "SELECT count(*) FROM CONTINENTS;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT countries.CountryId, countries.CountryName, Count(*) FROM countries GROUP BY countries.CountryId", "gold": "SELECT T1.ContId ,  T1.Continent ,  count(*) FROM CONTINENTS AS T1 JOIN COUNTRIES AS T2 ON T1.ContId  =  T2.Continent GROUP BY T1.ContId;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryId, countries.CountryName, Count(*) FROM countries GROUP BY countries.CountryId", "gold": "SELECT T1.ContId ,  T1.Continent ,  count(*) FROM CONTINENTS AS T1 JOIN COUNTRIES AS T2 ON T1.ContId  =  T2.Continent GROUP BY T1.ContId;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countries", "gold": "SELECT count(*) FROM COUNTRIES;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM countries", "gold": "SELECT count(*) FROM COUNTRIES;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT countries.CountryName, countries.CountryName, Count(*) FROM model_list JOIN countries GROUP BY countries.CountryName", "gold": "SELECT T1.FullName ,  T1.Id ,  count(*) FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT car_makers.FullName, car_makers.FullName, Count(*) FROM model_list JOIN car_makers GROUP BY car_makers.Id", "gold": "SELECT T1.FullName ,  T1.Id ,  count(*) FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT model_list.Model FROM cars_data JOIN model_list ORDER BY cars_data.Accelerate Asc LIMIT 1", "gold": "SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id ORDER BY T2.horsepower ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT model_list.Model FROM model_list JOIN cars_data ORDER BY cars_data.Accelerate Asc LIMIT 1", "gold": "SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id ORDER BY T2.horsepower ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT model_list.Model FROM model_list WHERE cars_data.Weight < (SELECT Avg(cars_data.Weight) FROM cars_data)", "gold": "SELECT T1.model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Weight  <  (SELECT avg(Weight) FROM CARS_DATA)", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT model_list.Model FROM model_list WHERE cars_data.Weight < (SELECT Avg(cars_data.Weight) FROM cars_data)", "gold": "SELECT T1.model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Weight  <  (SELECT avg(Weight) FROM CARS_DATA)", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName FROM cars_data JOIN countries WHERE cars_data.Year = 'terminal'", "gold": "SELECT DISTINCT T1.Maker FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker JOIN CAR_NAMES AS T3 ON T2.model  =  T3.model JOIN CARS_DATA AS T4 ON T3.MakeId  =  T4.id WHERE T4.year  =  '1970';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT continents.Continent FROM cars_data JOIN continents WHERE cars_data.Year = 'terminal'", "gold": "SELECT DISTINCT T1.Maker FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker JOIN CAR_NAMES AS T3 ON T2.model  =  T3.model JOIN CARS_DATA AS T4 ON T3.MakeId  =  T4.id WHERE T4.year  =  '1970';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName, cars_data.Accelerate FROM cars_data JOIN countries ORDER BY cars_data.Year Asc LIMIT 1", "gold": "SELECT T2.Make ,  T1.Year FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T1.Year  =  (SELECT min(YEAR) FROM CARS_DATA);", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT continents.Continent, cars_data.Year FROM cars_data JOIN continents ORDER BY cars_data.Year Asc LIMIT 1", "gold": "SELECT T2.Make ,  T1.Year FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T1.Year  =  (SELECT min(YEAR) FROM CARS_DATA);", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT DISTINCT model_list.Model FROM model_list JOIN cars_data WHERE cars_data.Year > 'terminal'", "gold": "SELECT DISTINCT T1.model FROM MODEL_LIST AS T1 JOIN CAR_NAMES AS T2 ON T1.model  =  T2.model JOIN CARS_DATA AS T3 ON T2.MakeId  =  T3.id WHERE T3.year  >  1980;", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT model_list.Model FROM model_list JOIN cars_data WHERE cars_data.Year > 'terminal'", "gold": "SELECT DISTINCT T1.model FROM MODEL_LIST AS T1 JOIN CAR_NAMES AS T2 ON T1.model  =  T2.model JOIN CARS_DATA AS T3 ON T2.MakeId  =  T3.id WHERE T3.year  >  1980;", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName, Count(*) FROM car_makers JOIN countries GROUP BY car_makers.Country", "gold": "SELECT T1.Continent ,  count(*) FROM CONTINENTS AS T1 JOIN COUNTRIES AS T2 ON T1.ContId  =  T2.continent JOIN car_makers AS T3 ON T2.CountryId  =  T3.Country GROUP BY T1.Continent;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName, Count(*) FROM countries JOIN car_makers GROUP BY car_makers.Country", "gold": "SELECT T1.Continent ,  count(*) FROM CONTINENTS AS T1 JOIN COUNTRIES AS T2 ON T1.ContId  =  T2.continent JOIN car_makers AS T3 ON T2.CountryId  =  T3.Country GROUP BY T1.Continent;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName FROM countries GROUP BY countries.CountryName ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.CountryName FROM CAR_MAKERS AS T1 JOIN COUNTRIES AS T2 ON T1.Country  =  T2.CountryId GROUP BY T1.Country ORDER BY Count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT countries.CountryName FROM countries JOIN car_makers GROUP BY car_makers.Country ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.CountryName FROM CAR_MAKERS AS T1 JOIN COUNTRIES AS T2 ON T1.Country  =  T2.CountryId GROUP BY T1.Country ORDER BY Count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(*), countries.CountryName FROM car_names JOIN countries GROUP BY countries.CountryName", "gold": "select count(*) ,  t2.fullname from model_list as t1 join car_makers as t2 on t1.maker  =  t2.id group by t2.id;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), car_makers.Id, car_makers.FullName FROM model_list JOIN car_makers GROUP BY car_makers.Id", "gold": "SELECT Count(*) ,  T2.FullName ,  T2.id FROM MODEL_LIST AS T1 JOIN CAR_MAKERS AS T2 ON T1.Maker  =  T2.Id GROUP BY T2.id;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT cars_data.MPG FROM car_names JOIN cars_data WHERE car_names.Make = 'terminal'", "gold": "SELECT T1.Accelerate FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T2.Make  =  'amc hornet sportabout (sw)';", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT cars_data.MPG FROM car_names JOIN cars_data WHERE car_names.Make = 'terminal'", "gold": "SELECT T1.Accelerate FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T2.Make  =  'amc hornet sportabout (sw)';", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM car_makers WHERE car_makers.Country = 'terminal'", "gold": "SELECT count(*) FROM CAR_MAKERS AS T1 JOIN COUNTRIES AS T2 ON T1.Country  =  T2.CountryId WHERE T2.CountryName  =  'france';", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countries WHERE countries.CountryName = 'terminal'", "gold": "SELECT count(*) FROM CAR_MAKERS AS T1 JOIN COUNTRIES AS T2 ON T1.Country  =  T2.CountryId WHERE T2.CountryName  =  'france';", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT model_list.Model) FROM model_list JOIN car_makers WHERE car_makers.Country = 'terminal'", "gold": "SELECT count(*) FROM MODEL_LIST AS T1 JOIN CAR_MAKERS AS T2 ON T1.Maker  =  T2.Id JOIN COUNTRIES AS T3 ON T2.Country  =  T3.CountryId WHERE T3.CountryName  =  'usa';", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT model_list.Model) FROM model_list JOIN car_makers WHERE car_makers.Country = 'terminal'", "gold": "SELECT count(*) FROM MODEL_LIST AS T1 JOIN CAR_MAKERS AS T2 ON T1.Maker  =  T2.Id JOIN COUNTRIES AS T3 ON T2.Country  =  T3.CountryId WHERE T3.CountryName  =  'usa';", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.MPG) FROM cars_data JOIN car_names WHERE car_names.Make LIKE 'terminal'", "gold": "SELECT avg(mpg) FROM CARS_DATA WHERE Cylinders  =  4;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Avg(cars_data.Cylinders), cars_data.Cylinders FROM cars_data WHERE cars_data.Horsepower = 'terminal' GROUP BY cars_data.Cylinders", "gold": "SELECT avg(mpg) FROM CARS_DATA WHERE Cylinders  =  4;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT cars_data.Weight FROM cars_data WHERE cars_data.Year = 'terminal'", "gold": "select min(weight) from cars_data where cylinders  =  8 and year  =  1974", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Min(cars_data.Weight) FROM cars_data WHERE cars_data.Year = 'terminal'", "gold": "select min(weight) from cars_data where cylinders  =  8 and year  =  1974", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT car_makers.Maker, model_list.Model FROM model_list JOIN car_makers", "gold": "SELECT Maker ,  Model FROM MODEL_LIST;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT car_makers.FullName, model_list.Model FROM model_list JOIN car_makers", "gold": "SELECT Maker ,  Model FROM MODEL_LIST;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT countries.CountryName, countries.CountryId FROM countries JOIN continents GROUP BY countries.CountryId HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.CountryName ,  T1.CountryId FROM COUNTRIES AS T1 JOIN CAR_MAKERS AS T2 ON T1.CountryId  =  T2.Country GROUP BY T1.CountryId HAVING count(*)  >=  1;", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT countries.CountryName, countries.CountryId FROM countries JOIN continents GROUP BY countries.CountryName HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.CountryName ,  T1.CountryId FROM COUNTRIES AS T1 JOIN CAR_MAKERS AS T2 ON T1.CountryId  =  T2.Country GROUP BY T1.CountryId HAVING count(*)  >=  1;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Horsepower > 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE horsepower  >  150;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Horsepower > 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE horsepower  >  150;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.Weight), cars_data.Year FROM cars_data GROUP BY cars_data.Year", "gold": "SELECT avg(Weight) ,  YEAR FROM CARS_DATA GROUP BY YEAR;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.Horsepower), cars_data.Year FROM cars_data GROUP BY cars_data.Year", "gold": "SELECT avg(Weight) ,  YEAR FROM CARS_DATA GROUP BY YEAR;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT car_makers.Country FROM car_makers GROUP BY car_makers.Country HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.CountryName FROM COUNTRIES AS T1 JOIN CONTINENTS AS T2 ON T1.Continent  =  T2.ContId JOIN CAR_MAKERS AS T3 ON T1.CountryId  =  T3.Country WHERE T2.Continent  =  'europe' GROUP BY T1.CountryName HAVING count(*)  >=  3;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT countries.CountryName FROM countries JOIN car_makers GROUP BY countries.CountryName HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.CountryName FROM COUNTRIES AS T1 JOIN CONTINENTS AS T2 ON T1.Continent  =  T2.ContId JOIN CAR_MAKERS AS T3 ON T1.CountryId  =  T3.Country WHERE T2.Continent  =  'europe' GROUP BY T1.CountryName HAVING count(*)  >=  3;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT Max(cars_data.Horsepower), Max(cars_data.Accelerate) FROM cars_data", "gold": "SELECT T2.horsepower ,  T1.Make FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.cylinders  =  3 ORDER BY T2.horsepower DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 0}}}, {"predicted": "SELECT Max(cars_data.Cylinders), countries.CountryName FROM car_makers JOIN countries GROUP BY countries.CountryName", "gold": "SELECT T2.horsepower ,  T1.Make FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.cylinders  =  3 ORDER BY T2.horsepower DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT car_names.Model FROM cars_data JOIN car_names ORDER BY cars_data.Cylinders Desc LIMIT 1", "gold": "SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id ORDER BY T2.mpg DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT model_list.Model FROM model_list JOIN cars_data ORDER BY cars_data.MPG Desc LIMIT 1", "gold": "select t1.model from car_names as t1 join cars_data as t2 on t1.makeid  =  t2.id order by t2.mpg desc limit 1;", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(cars_data.Horsepower) FROM cars_data WHERE cars_data.Year < 'terminal'", "gold": "SELECT avg(horsepower) FROM CARS_DATA WHERE YEAR  <  1980;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.Horsepower) FROM cars_data WHERE cars_data.Year < 'terminal'", "gold": "select avg(horsepower) from cars_data where year  <  1980;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.Edispl) FROM model_list JOIN cars_data WHERE model_list.Model = 'terminal'", "gold": "SELECT avg(T2.edispl) FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T1.Model  =  'volvo';", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(cars_data.Edispl) FROM cars_data", "gold": "SELECT avg(T2.edispl) FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T1.Model  =  'volvo';", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Max(cars_data.Horsepower), cars_data.Weight FROM cars_data GROUP BY cars_data.Weight", "gold": "SELECT max(Accelerate) ,  Cylinders FROM CARS_DATA GROUP BY Cylinders;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Max(cars_data.Accelerate), cars_data.Weight FROM cars_data GROUP BY cars_data.Weight", "gold": "SELECT max(Accelerate) ,  Cylinders FROM CARS_DATA GROUP BY Cylinders;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT model_list.Model, Count(*) FROM model_list GROUP BY model_list.Model ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Model FROM CAR_NAMES GROUP BY Model ORDER BY count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT model_list.Model FROM model_list GROUP BY model_list.Model ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Model FROM CAR_NAMES GROUP BY Model ORDER BY count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(*) FROM model_list WHERE model_list.ModelId > 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE Cylinders  >  4;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM model_list WHERE model_list.ModelId > 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE Cylinders  >  4;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Year = 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE YEAR  =  1980;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Year = 'terminal'", "gold": "SELECT count(*) FROM CARS_DATA WHERE YEAR  =  1980;", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM car_makers JOIN model_list WHERE car_makers.FullName = 'terminal'", "gold": "SELECT count(*) FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker WHERE T1.FullName  =  'American Motor Company';", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM model_list JOIN car_makers WHERE car_makers.FullName = 'terminal'", "gold": "SELECT count(*) FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker WHERE T1.FullName  =  'American Motor Company';", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT car_makers.FullName, car_makers.Id FROM model_list JOIN car_makers GROUP BY car_makers.Id HAVING Count(*) > 'terminal'", "gold": "SELECT T1.FullName ,  T1.Id FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id HAVING count(*)  >  3;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT car_makers.FullName, car_makers.Id FROM model_list JOIN car_makers GROUP BY model_list.Maker HAVING Count(*) > 'terminal'", "gold": "SELECT T1.FullName ,  T1.Id FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id HAVING count(*)  >  3;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT DISTINCT cars_data.MPG FROM continents JOIN countries WHERE car_makers.FullName = 'terminal' OR continents.Continent > 'terminal'", "gold": "SELECT DISTINCT T2.Model FROM CAR_NAMES AS T1 JOIN MODEL_LIST AS T2 ON T1.Model  =  T2.Model JOIN CAR_MAKERS AS T3 ON T2.Maker  =  T3.Id JOIN CARS_DATA AS T4 ON T1.MakeId  =  T4.Id WHERE T3.FullName  =  'General Motors' OR T4.weight  >  3500;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT model_list.Model FROM model_list JOIN car_makers JOIN cars_data WHERE car_makers.FullName = 'terminal' OR cars_data.Horsepower > 'terminal'", "gold": "SELECT DISTINCT T2.Model FROM CAR_NAMES AS T1 JOIN MODEL_LIST AS T2 ON T1.Model  =  T2.Model JOIN CAR_MAKERS AS T3 ON T2.Maker  =  T3.Id JOIN CARS_DATA AS T4 ON T1.MakeId  =  T4.Id WHERE T3.FullName  =  'General Motors' OR T4.weight  >  3500;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT cars_data.Year FROM cars_data WHERE cars_data.Horsepower < 'terminal' INTERSECT SELECT cars_data.Year FROM cars_data WHERE cars_data.Weight > 'terminal'", "gold": "select distinct year from cars_data where weight between 3000 and 4000;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT DISTINCT cars_data.Year FROM cars_data WHERE cars_data.Horsepower < 'terminal' INTERSECT SELECT cars_data.Year FROM cars_data WHERE cars_data.Weight > 'terminal'", "gold": "select distinct year from cars_data where weight between 3000 and 4000;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT cars_data.Accelerate FROM cars_data ORDER BY cars_data.Accelerate Desc LIMIT 1", "gold": "SELECT T1.horsepower FROM CARS_DATA AS T1 ORDER BY T1.accelerate DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT cars_data.Horsepower FROM cars_data ORDER BY cars_data.Accelerate Desc LIMIT 1", "gold": "SELECT T1.horsepower FROM CARS_DATA AS T1 ORDER BY T1.accelerate DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT model_list.Model, Count(*) FROM car_names JOIN model_list GROUP BY model_list.Model ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T1.cylinders FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T2.Model  =  'volvo' ORDER BY T1.accelerate ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(*), car_makers.FullName FROM car_names JOIN car_makers GROUP BY car_makers.FullName ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T1.cylinders FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T2.Model  =  'volvo' ORDER BY T1.accelerate ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Cylinders > (SELECT Max(cars_data.Cylinders) FROM cars_data)", "gold": "SELECT COUNT(*) FROM CARS_DATA WHERE Accelerate  >  ( SELECT Accelerate FROM CARS_DATA ORDER BY Horsepower DESC LIMIT 1 );", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM cars_data WHERE cars_data.Cylinders > (SELECT Max(cars_data.Accelerate) FROM cars_data)", "gold": "SELECT COUNT(*) FROM CARS_DATA WHERE Accelerate  >  ( SELECT Accelerate FROM CARS_DATA ORDER BY Horsepower DESC LIMIT 1 );", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM car_makers", "gold": "select count(*) from countries as t1 join car_makers as t2 on t1.countryid  =  t2.country group by t1.countryid having count(*)  >  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM (SELECT car_makers.Country FROM car_makers GROUP BY car_makers.Country HAVING Count(*) > 'terminal')", "gold": "select count(*) from countries as t1 join car_makers as t2 on t1.countryid  =  t2.country group by t1.countryid having count(*)  >  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM car_makers WHERE car_makers.FullName > 'terminal'", "gold": "SELECT COUNT(*) FROM CARS_DATA WHERE Cylinders  >  6;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM model_list WHERE model_list.ModelId > 'terminal'", "gold": "SELECT COUNT(*) FROM CARS_DATA WHERE Cylinders  >  6;", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT model_list.Model FROM cars_data WHERE cars_data.Cylinders = (SELECT Max(cars_data.Accelerate) FROM cars_data)", "gold": "SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Cylinders  =  4 ORDER BY T2.horsepower DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT model_list.Model FROM cars_data WHERE cars_data.Cylinders = (SELECT Max(cars_data.Accelerate) FROM cars_data)", "gold": "SELECT T1.Model FROM CAR_NAMES AS T1 JOIN CARS_DATA AS T2 ON T1.MakeId  =  T2.Id WHERE T2.Cylinders  =  4 ORDER BY T2.horsepower DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT car_makers.Maker FROM car_makers WHERE cars_data.Weight > (SELECT Min(cars_data.Weight) FROM cars_data)", "gold": "SELECT T2.MakeId ,  T2.Make FROM CARS_DATA AS T1 JOIN CAR_NAMES AS T2 ON T1.Id  =  T2.MakeId WHERE T1.Horsepower  >  (SELECT min(Horsepower) FROM CARS_DATA) AND T1.Cylinders  <=  3;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT car_makers.Id, continents.Continent FROM continents WHERE car_makers.Id NOT IN (SELECT cars_data.Id FROM cars_data WHERE cars_data.Horsepower < 'terminal')", "gold": "select t2.makeid ,  t2.make from cars_data as t1 join car_names as t2 on t1.id  =  t2.makeid where t1.horsepower  >  (select min(horsepower) from cars_data) and t1.cylinders  <  4;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT Max(cars_data.Cylinders) FROM cars_data WHERE cars_data.Year > 'terminal' OR cars_data.Year < 'terminal'", "gold": "select max(mpg) from cars_data where cylinders  =  8 or year  <  1980", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Max(cars_data.MPG) FROM cars_data WHERE cars_data.Year = 'terminal' OR cars_data.Year < 'terminal'", "gold": "select max(mpg) from cars_data where cylinders  =  8 or year  <  1980", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT cars_data.Weight FROM cars_data WHERE cars_data.Weight < (SELECT Max(cars_data.Weight) FROM car_makers WHERE car_makers.FullName = 'terminal')", "gold": "SELECT DISTINCT T1.model FROM MODEL_LIST AS T1 JOIN CAR_NAMES AS T2 ON T1.Model  =  T2.Model JOIN CARS_DATA AS T3 ON T2.MakeId  =  T3.Id JOIN CAR_MAKERS AS T4 ON T1.Maker  =  T4.Id WHERE T3.weight  <  3500 AND T4.FullName != 'Ford Motor Company';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT cars_data.Weight FROM cars_data WHERE cars_data.Weight < (SELECT Max(cars_data.Weight) FROM car_makers WHERE car_makers.FullName = 'terminal')", "gold": "SELECT DISTINCT T1.model FROM MODEL_LIST AS T1 JOIN CAR_NAMES AS T2 ON T1.Model  =  T2.Model JOIN CARS_DATA AS T3 ON T2.MakeId  =  T3.Id JOIN CAR_MAKERS AS T4 ON T1.Maker  =  T4.Id WHERE T3.weight  <  3500 AND T4.FullName != 'Ford Motor Company';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName FROM countries EXCEPT SELECT countries.CountryName FROM countries", "gold": "SELECT CountryName FROM countries EXCEPT SELECT T1.CountryName FROM countries AS T1 JOIN CAR_MAKERS AS T2 ON T1.countryId  =  T2.Country;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countries.CountryName FROM countries WHERE countries.CountryId NOT IN (SELECT car_makers.Country FROM car_makers)", "gold": "SELECT CountryName FROM countries EXCEPT SELECT T1.CountryName FROM countries AS T1 JOIN CAR_MAKERS AS T2 ON T1.countryId  =  T2.Country;", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT continents.ContId, continents.Continent FROM continents GROUP BY continents.ContId HAVING Count(*) >= 'terminal' INTERSECT SELECT continents.ContId, continents.Continent FROM model_list JOIN countries GROUP BY continents.ContId HAVING Count(*) >= 'terminal'", "gold": "select t1.id ,  t1.maker from car_makers as t1 join model_list as t2 on t1.id  =  t2.maker group by t1.id having count(*)  >=  2 intersect select t1.id ,  t1.maker from car_makers as t1 join model_list as t2 on t1.id  =  t2.maker join car_names as t3 on t2.model  =  t3.model group by t1.id having count(*)  >  3;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT car_makers.Id, car_makers.Country FROM continents GROUP BY car_makers.Id HAVING Count(*) > 'terminal' INTERSECT SELECT car_makers.Id, car_makers.Country FROM model_list GROUP BY model_list.ModelId HAVING Count(*) > 'terminal'", "gold": "SELECT T1.Id ,  T1.Maker FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker GROUP BY T1.Id HAVING count(*)  >=  2 INTERSECT SELECT T1.Id ,  T1.Maker FROM CAR_MAKERS AS T1 JOIN MODEL_LIST AS T2 ON T1.Id  =  T2.Maker JOIN CAR_NAMES AS T3 ON T2.model  =  T3.model GROUP BY T1.Id HAVING count(*)  >  3;", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT countries.CountryId, countries.CountryName FROM countries JOIN car_makers WHERE countries.CountryId > 'terminal' OR countries.CountryName > 'terminal'", "gold": "SELECT T1.countryId ,  T1.CountryName FROM Countries AS T1 JOIN CAR_MAKERS AS T2 ON T1.CountryId  =  T2.Country GROUP BY T1.countryId HAVING count(*)  >  3 UNION SELECT T1.countryId ,  T1.CountryName FROM Countries AS T1 JOIN CAR_MAKERS AS T2 ON T1.CountryId  =  T2.Country JOIN MODEL_LIST AS T3 ON T2.Id  =  T3.Maker WHERE T3.Model  =  'fiat';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT countries.CountryId, countries.CountryName FROM countries JOIN model_list GROUP BY countries.CountryId HAVING Count(*) > 'terminal' UNION SELECT countries.CountryId, countries.CountryName FROM countries JOIN car_makers WHERE model_list.Model = 'terminal'", "gold": "select t1.countryid ,  t1.countryname from countries as t1 join car_makers as t2 on t1.countryid  =  t2.country group by t1.countryid having count(*)  >  3 union select t1.countryid ,  t1.countryname from countries as t1 join car_makers as t2 on t1.countryid  =  t2.country join model_list as t3 on t2.id  =  t3.maker where t3.model  =  'fiat';", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT airlines.Country FROM airlines WHERE airlines.Airline = 'terminal'", "gold": "SELECT Country FROM AIRLINES WHERE Airline  =  \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Country FROM airlines WHERE airlines.Airline = 'terminal'", "gold": "SELECT Country FROM AIRLINES WHERE Airline  =  \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airlines JOIN airports WHERE airlines.Airline = 'terminal'", "gold": "SELECT Abbreviation FROM AIRLINES WHERE Airline  =  \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines WHERE airlines.Airline = 'terminal'", "gold": "SELECT Abbreviation FROM AIRLINES WHERE Airline  =  \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline, airlines.Airline FROM airlines WHERE airlines.Country = 'terminal'", "gold": "SELECT Airline ,  Abbreviation FROM AIRLINES WHERE Country  =  \"USA\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline, airlines.Airline FROM airlines WHERE airlines.Country = 'terminal'", "gold": "SELECT Airline ,  Abbreviation FROM AIRLINES WHERE Country  =  \"USA\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportCode, airports.AirportName FROM airports WHERE airports.City = 'terminal'", "gold": "SELECT AirportCode ,  AirportName FROM AIRPORTS WHERE city  =  \"Anthony\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportCode, airports.AirportName FROM airports WHERE airports.City = 'terminal'", "gold": "SELECT AirportCode ,  AirportName FROM AIRPORTS WHERE city  =  \"Anthony\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines", "gold": "SELECT count(*) FROM AIRLINES", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM airlines", "gold": "SELECT count(*) FROM AIRLINES", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM airports", "gold": "SELECT count(*) FROM AIRPORTS", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM airports", "gold": "SELECT count(*) FROM AIRPORTS", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM flights", "gold": "SELECT count(*) FROM FLIGHTS", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM flights", "gold": "SELECT count(*) FROM FLIGHTS", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT airlines.Airline FROM airlines WHERE airlines.Airline LIKE 'terminal'", "gold": "SELECT Airline FROM AIRLINES WHERE Abbreviation  =  \"UAL\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines WHERE airlines.Airline = 'terminal'", "gold": "SELECT Airline FROM AIRLINES WHERE Abbreviation  =  \"UAL\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines WHERE airlines.Country = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES WHERE Country  =  \"USA\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines WHERE airlines.Country = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES WHERE Country  =  \"USA\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.City, airports.Country FROM airports WHERE airports.AirportName = 'terminal'", "gold": "SELECT City ,  Country FROM AIRPORTS WHERE AirportName  =  \"Alton\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.City, airports.Country FROM airports WHERE airports.AirportName = 'terminal'", "gold": "SELECT City ,  Country FROM AIRPORTS WHERE AirportName  =  \"Alton\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT AirportName FROM AIRPORTS WHERE AirportCode  =  \"AKO\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT AirportName FROM AIRPORTS WHERE AirportCode  =  \"AKO\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airports WHERE airports.City = 'terminal'", "gold": "SELECT AirportName FROM AIRPORTS WHERE City = \"Aberdeen\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airports WHERE airports.City = 'terminal'", "gold": "SELECT AirportName FROM AIRPORTS WHERE City = \"Aberdeen\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS WHERE SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS WHERE SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM flights WHERE flights.DestAirport = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS WHERE DestAirport  =  \"ATO\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM flights WHERE flights.DestAirport = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS WHERE DestAirport  =  \"ATO\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.SourceAirport  =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.SourceAirport  =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' AND airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode JOIN AIRPORTS AS T3 ON T1.SourceAirport  =  T3.AirportCode WHERE T2.City  =  \"Ashley\" AND T3.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' AND airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode JOIN AIRPORTS AS T3 ON T1.SourceAirport  =  T3.AirportCode WHERE T2.City  =  \"Ashley\" AND T3.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines JOIN flights WHERE airlines.Airline = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRLINES AS T2 ON T1.Airline  =  T2.uid WHERE T2.Airline = \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines JOIN flights WHERE airlines.Airline = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRLINES AS T2 ON T1.Airline  =  T2.uid WHERE T2.Airline = \"JetBlue Airways\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airlines JOIN flights WHERE airlines.Airline = 'terminal' AND flights.SourceAirport = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T2.Airline  =  T1.uid WHERE T1.Airline  =  \"United Airlines\" AND T2.DestAirport  =  \"ASY\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.AirportName = 'terminal' AND airports.AirportCode = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T2.Airline  =  T1.uid WHERE T1.Airline  =  \"United Airlines\" AND T2.DestAirport  =  \"ASY\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.AirportName = 'terminal' AND flights.SourceAirport = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T2.Airline  =  T1.uid WHERE T1.Airline  =  \"United Airlines\" AND T2.SourceAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights JOIN airlines WHERE airlines.Airline = 'terminal' AND airports.AirportName = 'terminal'", "gold": "SELECT count(*) FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T2.Airline  =  T1.uid WHERE T1.Airline  =  \"United Airlines\" AND T2.SourceAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' AND airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode JOIN AIRLINES AS T3 ON T3.uid  =  T1.Airline WHERE T2.City  =  \"Aberdeen\" AND T3.Airline  =  \"United Airlines\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' AND airports.City = 'terminal'", "gold": "SELECT count(*) FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport  =  T2.AirportCode JOIN AIRLINES AS T3 ON T3.uid  =  T1.Airline WHERE T2.City  =  \"Aberdeen\" AND T3.Airline  =  \"United Airlines\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airports.City FROM airports JOIN flights GROUP BY airports.City ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.City FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport GROUP BY T1.City ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.City FROM airports JOIN flights GROUP BY airports.City ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.City FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport GROUP BY T1.City ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.City FROM airports JOIN flights GROUP BY airports.City ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.City FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.SourceAirport GROUP BY T1.City ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.City FROM airports GROUP BY airports.City ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.City FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.SourceAirport GROUP BY T1.City ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.AirportCode FROM airports JOIN flights GROUP BY flights.SourceAirport ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.AirportCode FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport OR T1.AirportCode  =  T2.SourceAirport GROUP BY T1.AirportCode ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT flights.SourceAirport FROM flights GROUP BY flights.SourceAirport ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.AirportCode FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport OR T1.AirportCode  =  T2.SourceAirport GROUP BY T1.AirportCode ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.AirportCode FROM airports JOIN flights GROUP BY flights.SourceAirport ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T1.AirportCode FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport OR T1.AirportCode  =  T2.SourceAirport GROUP BY T1.AirportCode ORDER BY count(*) LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.AirportCode FROM airports JOIN flights GROUP BY flights.SourceAirport ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T1.AirportCode FROM AIRPORTS AS T1 JOIN FLIGHTS AS T2 ON T1.AirportCode  =  T2.DestAirport OR T1.AirportCode  =  T2.SourceAirport GROUP BY T1.AirportCode ORDER BY count(*) LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights GROUP BY airlines.Airline ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights GROUP BY airlines.Airline ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airlines.Airline, airlines.Country FROM airlines JOIN flights GROUP BY airlines.Airline ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T1.Abbreviation ,  T1.Country FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline ORDER BY count(*) LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT airports.AirportName, airports.Country FROM airports WHERE airports.AirportName = 'terminal'", "gold": "SELECT T1.Abbreviation ,  T1.Country FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline ORDER BY count(*) LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.DestAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.DestAirport  =  \"AHD\"", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN airports JOIN flights WHERE airports.AirportCode = 'terminal' INTERSECT SELECT airlines.Airline FROM airlines JOIN airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"APG\" INTERSECT SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"CVO\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights JOIN airports WHERE airports.AirportCode = 'terminal' INTERSECT SELECT airlines.Airline FROM airlines JOIN airports WHERE airports.AirportCode = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"APG\" INTERSECT SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"CVO\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN airports WHERE flights.SourceAirport = 'terminal' EXCEPT SELECT airlines.Airline FROM airlines JOIN flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"CVO\" EXCEPT SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines WHERE airports.City = 'terminal' EXCEPT SELECT airlines.Airline FROM airlines JOIN airports WHERE airports.City = 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"CVO\" EXCEPT SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline WHERE T2.SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights GROUP BY airlines.Airline HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline HAVING count(*)  >  10", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines JOIN flights GROUP BY airlines.Airline HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline HAVING count(*)  >  10", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airlines.Airline FROM airlines WHERE airlines.Abbreviation < 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline HAVING count(*)  <  200", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT airlines.Airline FROM airlines GROUP BY airlines.Airline HAVING airlines.Abbreviation < 'terminal'", "gold": "SELECT T1.Airline FROM AIRLINES AS T1 JOIN FLIGHTS AS T2 ON T1.uid  =  T2.Airline GROUP BY T1.Airline HAVING count(*)  <  200", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT flights.FlightNo FROM airlines JOIN flights WHERE airlines.Airline = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRLINES AS T2 ON T2.uid  =  T1.Airline WHERE T2.Airline  =  \"United Airlines\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airlines JOIN flights WHERE airlines.Airline = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRLINES AS T2 ON T2.uid  =  T1.Airline WHERE T2.Airline  =  \"United Airlines\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM flights WHERE flights.SourceAirport = 'terminal'", "gold": "SELECT FlightNo FROM FLIGHTS WHERE SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT FlightNo FROM FLIGHTS WHERE SourceAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.AirportName = 'terminal'", "gold": "SELECT FlightNo FROM FLIGHTS WHERE DestAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.AirportName = 'terminal'", "gold": "SELECT FlightNo FROM FLIGHTS WHERE DestAirport  =  \"APG\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.SourceAirport   =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.SourceAirport   =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport   =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT flights.FlightNo FROM airports JOIN flights WHERE airports.City = 'terminal'", "gold": "SELECT T1.FlightNo FROM FLIGHTS AS T1 JOIN AIRPORTS AS T2 ON T1.DestAirport   =  T2.AirportCode WHERE T2.City  =  \"Aberdeen\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' OR airports.City = 'terminal'", "gold": "SELECT count(*) FROM Flights AS T1 JOIN Airports AS T2 ON T1.DestAirport  =  T2.AirportCode WHERE T2.city  =  \"Aberdeen\" OR T2.city  =  \"Abilene\"", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM airports JOIN flights WHERE airports.City = 'terminal' OR flights.Airline = 'terminal'", "gold": "SELECT count(*) FROM Flights AS T1 JOIN Airports AS T2 ON T1.DestAirport  =  T2.AirportCode WHERE T2.city  =  \"Aberdeen\" OR T2.city  =  \"Abilene\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT airports.AirportName FROM airports EXCEPT SELECT airports.AirportName FROM airports JOIN flights", "gold": "SELECT AirportName FROM Airports WHERE AirportCode NOT IN (SELECT SourceAirport FROM Flights UNION SELECT DestAirport FROM Flights)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT airports.AirportName FROM airports EXCEPT SELECT airports.AirportName FROM airports JOIN flights WHERE flights.FlightNo = 'terminal'", "gold": "SELECT AirportName FROM Airports WHERE AirportCode NOT IN (SELECT SourceAirport FROM Flights UNION SELECT DestAirport FROM Flights)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM employee", "gold": "SELECT count(*) FROM employee", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM employee", "gold": "SELECT count(*) FROM employee", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT employee.Name FROM employee ORDER BY employee.Age Asc", "gold": "SELECT name FROM employee ORDER BY age", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT employee.Name FROM employee ORDER BY employee.Age Asc", "gold": "SELECT name FROM employee ORDER BY age", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT employee.City, Count(*) FROM employee GROUP BY employee.City", "gold": "SELECT count(*) ,  city FROM employee GROUP BY city", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), employee.City FROM employee GROUP BY employee.City", "gold": "SELECT count(*) ,  city FROM employee GROUP BY city", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT employee.City FROM employee WHERE employee.Age > (SELECT Min(employee.Age) FROM employee)", "gold": "SELECT city FROM employee WHERE age  <  30 GROUP BY city HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT employee.City FROM employee WHERE employee.Age < 'terminal' GROUP BY employee.City ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT city FROM employee WHERE age  <  30 GROUP BY city HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 5}}}, {"predicted": "SELECT shop.Location, Count(*) FROM shop GROUP BY shop.Location", "gold": "SELECT count(*) ,  LOCATION FROM shop GROUP BY LOCATION", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT shop.Location, Count(*) FROM shop GROUP BY shop.Location", "gold": "SELECT count(*) ,  LOCATION FROM shop GROUP BY LOCATION", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT shop.Manager_name, shop.District FROM shop GROUP BY shop.Shop_ID ORDER BY shop.Number_products Desc LIMIT 1", "gold": "SELECT manager_name ,  district FROM shop ORDER BY number_products DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT shop.Manager_name, shop.District FROM shop JOIN hiring GROUP BY shop.Shop_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT manager_name ,  district FROM shop ORDER BY number_products DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT Min(shop.Number_products), Max(shop.Number_products) FROM shop", "gold": "SELECT min(Number_products) ,  max(Number_products) FROM shop", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Min(shop.Number_products), Max(shop.Number_products) FROM shop", "gold": "SELECT min(Number_products) ,  max(Number_products) FROM shop", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT shop.Name, shop.Location, shop.District FROM shop ORDER BY shop.Number_products Desc", "gold": "SELECT name ,  LOCATION ,  district FROM shop ORDER BY number_products DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT shop.Name, shop.Location, shop.District FROM shop ORDER BY shop.Number_products Desc", "gold": "SELECT name ,  LOCATION ,  district FROM shop ORDER BY number_products DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT shop.Name FROM shop WHERE shop.Number_products > (SELECT Avg(shop.Number_products) FROM shop)", "gold": "SELECT name FROM shop WHERE number_products  >  (SELECT avg(number_products) FROM shop)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT shop.Name FROM shop WHERE shop.Number_products > (SELECT Avg(shop.Number_products) FROM shop)", "gold": "SELECT name FROM shop WHERE number_products  >  (SELECT avg(number_products) FROM shop)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT employee.Name FROM employee JOIN evaluation ORDER BY evaluation.Year_awarded Desc LIMIT 1", "gold": "SELECT t1.name FROM employee AS t1 JOIN evaluation AS t2 ON t1.Employee_ID  =  t2.Employee_ID GROUP BY t2.Employee_ID ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT employee.Name FROM employee JOIN hiring GROUP BY hiring.Employee_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT t1.name FROM employee AS t1 JOIN evaluation AS t2 ON t1.Employee_ID  =  t2.Employee_ID GROUP BY t2.Employee_ID ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT employee.Name FROM employee JOIN evaluation ORDER BY evaluation.Bonus Desc LIMIT 1", "gold": "SELECT t1.name FROM employee AS t1 JOIN evaluation AS t2 ON t1.Employee_ID  =  t2.Employee_ID ORDER BY t2.bonus DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT employee.Name FROM employee JOIN evaluation ORDER BY evaluation.Bonus Desc LIMIT 1", "gold": "SELECT t1.name FROM employee AS t1 JOIN evaluation AS t2 ON t1.Employee_ID  =  t2.Employee_ID ORDER BY t2.bonus DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT employee.Name FROM employee WHERE employee.Employee_ID NOT IN (SELECT evaluation.Employee_ID FROM evaluation)", "gold": "SELECT name FROM employee WHERE Employee_ID NOT IN (SELECT Employee_ID FROM evaluation)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT employee.Name FROM employee WHERE employee.Employee_ID NOT IN (SELECT evaluation.Employee_ID FROM evaluation)", "gold": "SELECT name FROM employee WHERE Employee_ID NOT IN (SELECT Employee_ID FROM evaluation)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT shop.Name FROM shop JOIN hiring GROUP BY hiring.Shop_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT t2.name FROM hiring AS t1 JOIN shop AS t2 ON t1.shop_id  =  t2.shop_id GROUP BY t1.shop_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT shop.Name FROM shop ORDER BY shop.Number_products Desc LIMIT 1", "gold": "SELECT t2.name FROM hiring AS t1 JOIN shop AS t2 ON t1.shop_id  =  t2.shop_id GROUP BY t1.shop_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT shop.Name FROM shop WHERE shop.Shop_ID NOT IN (SELECT hiring.Shop_ID FROM hiring)", "gold": "SELECT name FROM shop WHERE shop_id NOT IN (SELECT shop_id FROM hiring)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT shop.Name FROM shop WHERE shop.Shop_ID NOT IN (SELECT hiring.Shop_ID FROM hiring)", "gold": "SELECT name FROM shop WHERE shop_id NOT IN (SELECT shop_id FROM hiring)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*), employee.Employee_ID FROM employee JOIN hiring GROUP BY employee.Employee_ID", "gold": "SELECT count(*) ,  t2.name FROM hiring AS t1 JOIN shop AS t2 ON t1.shop_id  =  t2.shop_id GROUP BY t2.name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), shop.Name FROM shop JOIN hiring GROUP BY shop.Shop_ID", "gold": "SELECT count(*) ,  t2.name FROM hiring AS t1 JOIN shop AS t2 ON t1.shop_id  =  t2.shop_id GROUP BY t2.name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(evaluation.Bonus) FROM evaluation", "gold": "SELECT sum(bonus) FROM evaluation", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Sum(evaluation.Bonus) FROM evaluation", "gold": "SELECT sum(bonus) FROM evaluation", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT * FROM employee", "gold": "SELECT * FROM hiring", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT * FROM employee", "gold": "SELECT * FROM hiring", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT shop.District FROM shop WHERE shop.Number_products < 'terminal' INTERSECT SELECT shop.District FROM shop WHERE shop.Number_products > 'terminal'", "gold": "SELECT district FROM shop WHERE Number_products  <  3000 INTERSECT SELECT district FROM shop WHERE Number_products  >  10000", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT shop.District FROM shop WHERE shop.Number_products < 'terminal' INTERSECT SELECT shop.District FROM shop WHERE shop.Number_products > 'terminal'", "gold": "SELECT district FROM shop WHERE Number_products  <  3000 INTERSECT SELECT district FROM shop WHERE Number_products  >  10000", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(DISTINCT shop.Location) FROM shop", "gold": "SELECT count(DISTINCT LOCATION) FROM shop", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT shop.Location) FROM shop", "gold": "SELECT count(DISTINCT LOCATION) FROM shop", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents", "gold": "SELECT count(*) FROM Documents", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents", "gold": "SELECT count(*) FROM Documents", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name, Documents.Document_Description FROM Documents", "gold": "SELECT document_id ,  document_name ,  document_description FROM Documents", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name, Documents.Document_Description FROM Documents", "gold": "SELECT document_id ,  document_name ,  document_description FROM Documents", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Documents.Document_Name, Documents.Document_ID FROM Documents WHERE Documents.Document_Description LIKE 'terminal'", "gold": "SELECT document_name ,  template_id FROM Documents WHERE Document_Description LIKE \"%w%\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_Name, Documents.Document_ID FROM Documents WHERE Documents.Document_Description LIKE 'terminal'", "gold": "SELECT document_name ,  template_id FROM Documents WHERE Document_Description LIKE \"%w%\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Template_ID, Documents.Document_Description FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT document_id ,  template_id ,  Document_Description FROM Documents WHERE document_name  =  \"Robbin CV\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Template_ID, Documents.Document_Description FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT document_id ,  template_id ,  Document_Description FROM Documents WHERE document_name  =  \"Robbin CV\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT Documents.Document_Description) FROM Documents", "gold": "SELECT count(DISTINCT template_id) FROM Documents", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Documents.Document_Name) FROM Documents", "gold": "SELECT count(DISTINCT template_id) FROM Documents", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents JOIN Templates WHERE Templates.Template_Type_Code = 'terminal'", "gold": "SELECT count(*) FROM Documents AS T1 JOIN Templates AS T2 ON T1.Template_ID  =  T2.Template_ID WHERE T2.Template_Type_Code  =  'PPT'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Documents JOIN Templates WHERE Templates.Template_Type_Code = 'terminal'", "gold": "SELECT count(*) FROM Documents AS T1 JOIN Templates AS T2 ON T1.Template_ID  =  T2.Template_ID WHERE T2.Template_Type_Code  =  'PPT'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Template_ID, Count(*) FROM Documents GROUP BY Documents.Template_ID", "gold": "SELECT template_id ,  count(*) FROM Documents GROUP BY template_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Template_ID, Count(*) FROM Documents GROUP BY Documents.Template_ID", "gold": "SELECT template_id ,  count(*) FROM Documents GROUP BY template_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code, Templates.Template_Type_Code FROM Documents JOIN Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.template_id ,  T2.Template_Type_Code FROM Documents AS T1 JOIN Templates AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Template_ID, Templates.Template_Type_Code FROM Documents JOIN Templates GROUP BY Documents.Template_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.template_id ,  T2.Template_Type_Code FROM Documents AS T1 JOIN Templates AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Template_ID FROM Documents GROUP BY Documents.Template_ID HAVING Count(*) > 'terminal'", "gold": "SELECT template_id FROM Documents GROUP BY template_id HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Template_ID FROM Documents GROUP BY Documents.Template_ID HAVING Count(*) > 'terminal'", "gold": "SELECT template_id FROM Documents GROUP BY template_id HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Templates.Template_ID FROM Templates EXCEPT SELECT Documents.Template_ID FROM Documents", "gold": "SELECT template_id FROM Templates EXCEPT SELECT template_id FROM Documents", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_ID FROM Templates EXCEPT SELECT Documents.Template_ID FROM Documents", "gold": "SELECT template_id FROM Templates EXCEPT SELECT template_id FROM Documents", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Templates", "gold": "SELECT count(*) FROM Templates", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents", "gold": "SELECT count(*) FROM Templates", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Templates.Template_ID, Templates.Version_Number, Templates.Template_Type_Code FROM Templates", "gold": "SELECT template_id ,  version_number ,  template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Templates.Template_ID, Templates.Version_Number, Templates.Template_Type_Code FROM Templates", "gold": "SELECT template_id ,  version_number ,  template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT Templates.Template_Type_Code FROM Templates", "gold": "SELECT DISTINCT template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code FROM Ref_Template_Types", "gold": "SELECT DISTINCT template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Templates.Template_ID FROM Templates WHERE Templates.Template_Type_Code = 'terminal' OR Templates.Template_Type_Code = 'terminal'", "gold": "SELECT template_id FROM Templates WHERE template_type_code  =  \"PP\" OR template_type_code  =  \"PPT\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Templates.Template_ID FROM Templates WHERE Templates.Template_Type_Code = 'terminal' OR Templates.Template_Type_Code = 'terminal'", "gold": "SELECT template_id FROM Templates WHERE template_type_code  =  \"PP\" OR template_type_code  =  \"PPT\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Code = 'terminal'", "gold": "SELECT count(*) FROM Templates WHERE template_type_code  =  \"CV\"", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Templates WHERE Templates.Template_Type_Code = 'terminal'", "gold": "SELECT count(*) FROM Templates WHERE template_type_code  =  \"CV\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Version_Number, Templates.Template_Type_Code FROM Templates WHERE Templates.Date_Effective_To > 'terminal'", "gold": "SELECT version_number ,  template_type_code FROM Templates WHERE version_number  >  5", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Version_Number, Templates.Template_Type_Code FROM Templates WHERE Templates.Template_Type_Code > 'terminal'", "gold": "SELECT version_number ,  template_type_code FROM Templates WHERE version_number  >  5", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code, Count(*) FROM Templates GROUP BY Templates.Template_Type_Code", "gold": "SELECT template_type_code ,  count(*) FROM Templates GROUP BY template_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code, Count(*) FROM Templates GROUP BY Templates.Template_Type_Code", "gold": "SELECT template_type_code ,  count(*) FROM Templates GROUP BY template_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT template_type_code FROM Templates GROUP BY template_type_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT template_type_code FROM Templates GROUP BY template_type_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code HAVING Count(*) < 'terminal'", "gold": "SELECT template_type_code FROM Templates GROUP BY template_type_code HAVING count(*)  <  3", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code HAVING Count(*) < 'terminal'", "gold": "SELECT template_type_code FROM Templates GROUP BY template_type_code HAVING count(*)  <  3", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Templates.Version_Number, Templates.Template_Type_Code FROM Templates ORDER BY Templates.Version_Number Asc LIMIT 1", "gold": "SELECT min(Version_Number) ,  template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 3}}}, {"predicted": "SELECT Templates.Version_Number, Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT min(Version_Number) ,  template_type_code FROM Templates", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 4}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Documents JOIN Templates WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id WHERE T2.document_name  =  \"Data base\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Documents JOIN Templates WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id WHERE T2.document_name  =  \"Data base\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_Name FROM Documents JOIN Templates WHERE Templates.Template_Type_Code = 'terminal'", "gold": "SELECT T2.document_name FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id WHERE T1.template_type_code  =  \"BK\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_Name FROM Documents JOIN Templates WHERE Templates.Template_Type_Code = 'terminal'", "gold": "SELECT T2.document_name FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id WHERE T1.template_type_code  =  \"BK\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code, Count(*) FROM Documents JOIN Templates GROUP BY Templates.Template_Type_Code", "gold": "SELECT T1.template_type_code ,  count(*) FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code, Count(*) FROM Templates GROUP BY Templates.Template_Type_Code", "gold": "SELECT T1.template_type_code ,  count(*) FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_type_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates GROUP BY Templates.Template_Type_Code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id GROUP BY T1.template_type_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates EXCEPT SELECT Templates.Template_Type_Code FROM Templates", "gold": "SELECT template_type_code FROM Templates EXCEPT SELECT template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Type_Code FROM Templates EXCEPT SELECT Templates.Template_Type_Code FROM Templates", "gold": "SELECT template_type_code FROM Templates EXCEPT SELECT template_type_code FROM Templates AS T1 JOIN Documents AS T2 ON T1.template_id  =  T2.template_id", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code, Ref_Template_Types.Template_Type_Description FROM Ref_Template_Types", "gold": "SELECT template_type_code ,  template_type_description FROM Ref_template_types", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code, Ref_Template_Types.Template_Type_Description FROM Ref_Template_Types", "gold": "SELECT template_type_code ,  template_type_description FROM Ref_template_types", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Description FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Code = 'terminal'", "gold": "SELECT template_type_description FROM Ref_template_types WHERE template_type_code  =  \"AD\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Description FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Code = 'terminal'", "gold": "SELECT template_type_description FROM Ref_template_types WHERE template_type_code  =  \"AD\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Description = 'terminal'", "gold": "SELECT template_type_code FROM Ref_template_types WHERE template_type_description  =  \"Book\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Description = 'terminal'", "gold": "SELECT template_type_code FROM Ref_template_types WHERE template_type_description  =  \"Book\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT Ref_Template_Types.Template_Type_Description FROM Ref_Template_Types JOIN Templates", "gold": "SELECT DISTINCT T1.template_type_description FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code JOIN Documents AS T3 ON T2.Template_ID  =  T3.template_ID", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT Documents.Document_Description FROM Documents", "gold": "SELECT DISTINCT T1.template_type_description FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code JOIN Documents AS T3 ON T2.Template_ID  =  T3.template_ID", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Ref_Template_Types.Template_Type_Code FROM Ref_Template_Types WHERE Ref_Template_Types.Template_Type_Description = 'terminal'", "gold": "SELECT T2.template_id FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code WHERE T1.template_type_description  =  \"Presentation\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Template_ID FROM Documents WHERE Documents.Document_Description = 'terminal'", "gold": "SELECT T2.template_id FROM Ref_template_types AS T1 JOIN Templates AS T2 ON T1.template_type_code  = T2.template_type_code WHERE T1.template_type_description  =  \"Presentation\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Documents", "gold": "SELECT count(*) FROM Paragraphs", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents", "gold": "SELECT count(*) FROM Paragraphs", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT count(*) FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_ID  =  T2.document_ID WHERE T2.document_name  =  'Summer Show'", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT count(*) FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_ID  =  T2.document_ID WHERE T2.document_name  =  'Summer Show'", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Templates.Template_Details FROM Documents JOIN Templates WHERE Documents.Document_Name LIKE 'terminal'", "gold": "select other_details from paragraphs where paragraph_text like 'korea'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Templates.Template_Details FROM Documents JOIN Templates WHERE Documents.Document_Name LIKE 'terminal'", "gold": "select other_details from paragraphs where paragraph_text like 'korea'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID, Templates.Template_Details FROM Documents JOIN Templates WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.paragraph_id ,   T1.paragraph_text FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id WHERE T2.Document_Name  =  'Welcome to NY'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Template_ID FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.paragraph_id ,   T1.paragraph_text FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id WHERE T2.Document_Name  =  'Welcome to NY'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_Description FROM Documents WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.paragraph_text FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id WHERE T2.document_name  =  \"Customer reviews\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_Description FROM Documents JOIN Paragraphs WHERE Documents.Document_Name = 'terminal'", "gold": "SELECT T1.paragraph_text FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id WHERE T2.document_name  =  \"Customer reviews\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_ID, Count(*) FROM Paragraphs JOIN Documents GROUP BY Documents.Document_ID", "gold": "SELECT document_id ,  count(*) FROM Paragraphs GROUP BY document_id ORDER BY document_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_ID, Count(*) FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID ORDER BY Count(*) Asc", "gold": "SELECT document_id ,  count(*) FROM Paragraphs GROUP BY document_id ORDER BY document_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name, Documents.Document_Name FROM Documents JOIN Paragraphs", "gold": "SELECT T1.document_id ,  T2.document_name ,  count(*) FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id GROUP BY T1.document_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name, Count(*) FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID", "gold": "SELECT T1.document_id ,  T2.document_name ,  count(*) FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id GROUP BY T1.document_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID HAVING Count(*) >= 'terminal'", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID HAVING Count(*) >= 'terminal'", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.document_id ,  T2.document_name FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id GROUP BY T1.document_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Document_ID, Documents.Document_Name FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.document_id ,  T2.document_name FROM Paragraphs AS T1 JOIN Documents AS T2 ON T1.document_id  =  T2.document_id GROUP BY T1.document_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id ORDER BY count(*) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Document_ID FROM Paragraphs JOIN Documents GROUP BY Paragraphs.Document_ID ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id ORDER BY count(*) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents GROUP BY Documents.Document_ID HAVING Count(*) BETWEEN 'terminal' AND 'terminal'", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id HAVING count(*) BETWEEN 1 AND 2", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents JOIN Paragraphs GROUP BY Documents.Document_ID HAVING Count(*) BETWEEN 'terminal' AND 'terminal'", "gold": "SELECT document_id FROM Paragraphs GROUP BY document_id HAVING count(*) BETWEEN 1 AND 2", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents WHERE Documents.Document_Name LIKE 'terminal' INTERSECT SELECT Documents.Document_ID FROM Documents JOIN Paragraphs WHERE Documents.Document_Name LIKE 'terminal'", "gold": "SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Brazil' INTERSECT SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Ireland'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Documents.Document_ID FROM Documents WHERE Documents.Document_Name LIKE 'terminal' INTERSECT SELECT Documents.Document_ID FROM Documents WHERE Documents.Document_Name LIKE 'terminal'", "gold": "SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Brazil' INTERSECT SELECT document_id FROM Paragraphs WHERE paragraph_text  =  'Ireland'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM teacher", "gold": "SELECT count(*) FROM teacher", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM teacher", "gold": "SELECT count(*) FROM teacher", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT teacher.Name FROM teacher ORDER BY teacher.Age Asc", "gold": "SELECT Name FROM teacher ORDER BY Age ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name FROM teacher ORDER BY teacher.Age Asc", "gold": "SELECT Name FROM teacher ORDER BY Age ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Age, teacher.Hometown FROM teacher", "gold": "SELECT Age ,  Hometown FROM teacher", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT teacher.Age, teacher.Hometown FROM teacher", "gold": "SELECT Age ,  Hometown FROM teacher", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Hometown != 'terminal'", "gold": "select name from teacher where hometown != \"little lever urban district\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Hometown != 'terminal'", "gold": "select name from teacher where hometown != \"little lever urban district\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Age = 'terminal' OR teacher.Age = 'terminal'", "gold": "SELECT Name FROM teacher WHERE Age  =  32 OR Age  =  33", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Age = 'terminal' OR teacher.Age = 'terminal'", "gold": "SELECT Name FROM teacher WHERE Age  =  32 OR Age  =  33", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Hometown FROM teacher ORDER BY teacher.Age Asc LIMIT 1", "gold": "SELECT Hometown FROM teacher ORDER BY Age ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT teacher.Hometown FROM teacher WHERE teacher.Age = (SELECT Min(teacher.Age) FROM teacher)", "gold": "SELECT Hometown FROM teacher ORDER BY Age ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT teacher.Hometown, Count(*) FROM teacher GROUP BY teacher.Hometown", "gold": "SELECT Hometown ,  COUNT(*) FROM teacher GROUP BY Hometown", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Hometown, Count(*) FROM teacher GROUP BY teacher.Hometown", "gold": "SELECT Hometown ,  COUNT(*) FROM teacher GROUP BY Hometown", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Hometown FROM teacher GROUP BY teacher.Hometown ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Hometown FROM teacher GROUP BY Hometown ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT teacher.Hometown FROM teacher GROUP BY teacher.Hometown ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Hometown FROM teacher GROUP BY Hometown ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT teacher.Hometown FROM teacher GROUP BY teacher.Hometown HAVING Count(*) >= 'terminal'", "gold": "SELECT Hometown FROM teacher GROUP BY Hometown HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Hometown FROM teacher GROUP BY teacher.Hometown HAVING Count(*) >= 'terminal'", "gold": "SELECT Hometown FROM teacher GROUP BY Hometown HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name, course.Course FROM course_arrange JOIN course JOIN teacher", "gold": "SELECT T3.Name ,  T2.Course FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT teacher.Name, course.Course FROM course_arrange JOIN teacher JOIN course", "gold": "SELECT T3.Name ,  T2.Course FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT teacher.Name, course.Course FROM course_arrange JOIN teacher JOIN course ORDER BY teacher.Name Asc", "gold": "SELECT T3.Name ,  T2.Course FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID ORDER BY T3.Name", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name, course.Course FROM course_arrange JOIN course JOIN teacher ORDER BY teacher.Name Asc", "gold": "SELECT T3.Name ,  T2.Course FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID ORDER BY T3.Name", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name FROM course_arrange JOIN teacher WHERE course_arrange.Grade = 'terminal'", "gold": "SELECT T3.Name FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID WHERE T2.Course  =  \"Math\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name FROM course_arrange JOIN teacher WHERE course_arrange.Grade = 'terminal'", "gold": "SELECT T3.Name FROM course_arrange AS T1 JOIN course AS T2 ON T1.Course_ID  =  T2.Course_ID JOIN teacher AS T3 ON T1.Teacher_ID  =  T3.Teacher_ID WHERE T2.Course  =  \"Math\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name, Count(*) FROM course_arrange JOIN teacher GROUP BY course_arrange.Teacher_ID", "gold": "SELECT T2.Name ,  COUNT(*) FROM course_arrange AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID GROUP BY T2.Name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name, Count(*) FROM course_arrange JOIN teacher GROUP BY course_arrange.Teacher_ID", "gold": "SELECT T2.Name ,  COUNT(*) FROM course_arrange AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID GROUP BY T2.Name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT teacher.Name FROM course_arrange JOIN teacher GROUP BY course_arrange.Teacher_ID HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.Name FROM course_arrange AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID GROUP BY T2.Name HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name FROM course_arrange JOIN teacher GROUP BY course_arrange.Teacher_ID HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.Name FROM course_arrange AS T1 JOIN teacher AS T2 ON T1.Teacher_ID  =  T2.Teacher_ID GROUP BY T2.Name HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Teacher_ID NOT IN (SELECT course_arrange.Teacher_ID FROM course_arrange)", "gold": "SELECT Name FROM teacher WHERE Teacher_id NOT IN (SELECT Teacher_id FROM course_arrange)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT teacher.Name FROM teacher WHERE teacher.Teacher_ID NOT IN (SELECT course_arrange.Teacher_ID FROM course_arrange)", "gold": "SELECT Name FROM teacher WHERE Teacher_id NOT IN (SELECT Teacher_id FROM course_arrange)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM visitor WHERE visitor.Age < 'terminal'", "gold": "SELECT count(*) FROM visitor WHERE age  <  30", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT visitor.Name FROM visitor WHERE visitor.Level_of_membership > 'terminal' ORDER BY visitor.Level_of_membership Asc", "gold": "SELECT name FROM visitor WHERE Level_of_membership  >  4 ORDER BY Level_of_membership DESC", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(visitor.Age) FROM visitor WHERE visitor.ID NOT IN (SELECT visit.visitor_ID FROM visit GROUP BY visit.visitor_ID HAVING Max(visitor.Level_of_membership) > 'terminal')", "gold": "SELECT avg(age) FROM visitor WHERE Level_of_membership  <=  4", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT visitor.Name, visitor.Age FROM visitor WHERE visitor.Age > 'terminal' ORDER BY visitor.Age Asc", "gold": "SELECT name ,  Level_of_membership FROM visitor WHERE Level_of_membership  >  4 ORDER BY age DESC", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT museum.Museum_ID, museum.Name FROM museum ORDER BY museum.Num_of_Staff Desc LIMIT 1", "gold": "SELECT museum_id ,  name FROM museum ORDER BY num_of_staff DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(museum.Num_of_Staff) FROM museum WHERE museum.Open_Year < 'terminal'", "gold": "SELECT avg(num_of_staff) FROM museum WHERE open_year  <  2009", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT museum.Open_Year, museum.Num_of_Staff FROM museum WHERE museum.Name = 'terminal'", "gold": "SELECT Num_of_Staff ,  Open_Year FROM museum WHERE name  =  'Plaza Museum'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT museum.Name FROM museum WHERE museum.Num_of_Staff > (SELECT Min(museum.Num_of_Staff) FROM museum WHERE museum.Open_Year > 'terminal')", "gold": "SELECT name FROM museum WHERE num_of_staff  >  (SELECT min(num_of_staff) FROM museum WHERE open_year  >  2010)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT visitor.ID, visitor.Name, visitor.Age FROM visitor JOIN visit GROUP BY visitor.ID HAVING Count(*) > 'terminal'", "gold": "SELECT t1.id ,  t1.name ,  t1.age FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id GROUP BY t1.id HAVING count(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT visit.visitor_ID, visitor.Name, Sum(visit.Total_spent) FROM visit JOIN visitor GROUP BY visit.visitor_ID ORDER BY Sum(visit.Total_spent) Desc LIMIT 1", "gold": "SELECT t2.visitor_id ,  t1.name ,  t1.Level_of_membership FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id GROUP BY t2.visitor_id ORDER BY sum(t2.Total_spent) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT museum.Museum_ID, museum.Name FROM visit JOIN museum GROUP BY visit.Museum_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT t2.Museum_ID ,  t1.name FROM museum AS t1 JOIN visit AS t2 ON t1.Museum_ID  =  t2.Museum_ID GROUP BY t2.Museum_ID ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT museum.Name FROM museum WHERE museum.Museum_ID NOT IN (SELECT visit.Museum_ID FROM visit)", "gold": "SELECT name FROM museum WHERE Museum_ID NOT IN (SELECT museum_id FROM visit)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT visitor.Name, visitor.Age FROM visitor JOIN visit GROUP BY visit.visitor_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT t1.name ,  t1.age FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id ORDER BY t2.num_of_ticket DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT Avg(visit.Num_of_Ticket), Max(visit.Total_spent) FROM visit", "gold": "SELECT avg(num_of_ticket) ,  max(num_of_ticket) FROM visit", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Sum(visit.Total_spent) FROM visit WHERE visit.Museum_ID = 'terminal'", "gold": "SELECT sum(t2.Total_spent) FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id WHERE t1.Level_of_membership  =  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT museum.Name FROM museum WHERE museum.Open_Year < 'terminal' INTERSECT SELECT museum.Name FROM museum WHERE museum.Open_Year > 'terminal'", "gold": "SELECT t1.name FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id JOIN museum AS t3 ON t3.Museum_ID  =  t2.Museum_ID WHERE t3.open_year  <  2009 INTERSECT SELECT t1.name FROM visitor AS t1 JOIN visit AS t2 ON t1.id  =  t2.visitor_id JOIN museum AS t3 ON t3.Museum_ID  =  t2.Museum_ID WHERE t3.open_year  >  2011", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM museum WHERE museum.Museum_ID NOT IN (SELECT museum.Museum_ID FROM museum WHERE museum.Open_Year > 'terminal')", "gold": "SELECT count(*) FROM visitor WHERE id NOT IN (SELECT t2.visitor_id FROM museum AS t1 JOIN visit AS t2 ON t1.Museum_ID  =  t2.Museum_ID WHERE t1.open_year  >  2010)", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM museum WHERE museum.Open_Year > 'terminal' OR museum.Open_Year < 'terminal'", "gold": "SELECT count(*) FROM museum WHERE open_year  >  2013 OR open_year  <  2008", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM players", "gold": "SELECT count(*) FROM players", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM players", "gold": "SELECT count(*) FROM players", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM players", "gold": "SELECT count(*) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM players", "gold": "SELECT count(*) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT players.first_name, players.birth_date FROM players WHERE players.country_code = 'terminal'", "gold": "SELECT first_name ,  birth_date FROM players WHERE country_code  =  'USA'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.first_name, players.birth_date FROM players WHERE players.country_code = 'terminal'", "gold": "SELECT first_name ,  birth_date FROM players WHERE country_code  =  'USA'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(matches.winner_age), matches.loser_age FROM matches", "gold": "SELECT avg(loser_age) ,  avg(winner_age) FROM matches", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(matches.winner_age), matches.loser_age FROM matches", "gold": "SELECT avg(loser_age) ,  avg(winner_age) FROM matches", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(matches.score) FROM matches", "gold": "SELECT avg(winner_rank) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(matches.score) FROM matches", "gold": "SELECT avg(winner_rank) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT rankings.ranking FROM rankings", "gold": "SELECT min(loser_rank) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT matches.score FROM matches", "gold": "SELECT min(loser_rank) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT players.country_code) FROM players", "gold": "SELECT count(DISTINCT country_code) FROM players", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT players.country_code) FROM players", "gold": "SELECT count(DISTINCT country_code) FROM players", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT players.first_name) FROM players", "gold": "SELECT count(DISTINCT loser_name) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT players.first_name) FROM players", "gold": "SELECT count(DISTINCT loser_name) FROM matches", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT matches.winner_name FROM matches GROUP BY matches.winner_name HAVING Count(*) > 'terminal'", "gold": "SELECT tourney_name FROM matches GROUP BY tourney_name HAVING count(*)  >  10", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT rankings.tours FROM rankings JOIN matches GROUP BY rankings.tours HAVING Count(*) > 'terminal'", "gold": "SELECT tourney_name FROM matches GROUP BY tourney_name HAVING count(*)  >  10", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name FROM players JOIN matches WHERE matches.year = 'terminal' UNION SELECT matches.winner_name FROM matches WHERE matches.year = 'terminal'", "gold": "SELECT winner_name FROM matches WHERE YEAR  =  2013 INTERSECT SELECT winner_name FROM matches WHERE YEAR  =  2016", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name FROM players JOIN matches WHERE matches.year LIKE 'terminal' UNION SELECT players.first_name FROM players JOIN matches WHERE matches.year = 'terminal'", "gold": "SELECT winner_name FROM matches WHERE YEAR  =  2013 INTERSECT SELECT winner_name FROM matches WHERE YEAR  =  2016", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM matches WHERE matches.year = 'terminal' OR matches.year = 'terminal'", "gold": "SELECT count(*) FROM matches WHERE YEAR  =  2013 OR YEAR  =  2016", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM matches WHERE matches.year = 'terminal' OR matches.year = 'terminal'", "gold": "SELECT count(*) FROM matches WHERE YEAR  =  2013 OR YEAR  =  2016", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.country_code, matches.loser_name FROM players JOIN matches WHERE matches.tourney_name = 'terminal' INTERSECT SELECT players.country_code, matches.loser_name FROM players JOIN matches WHERE matches.tourney_name = 'terminal'", "gold": "SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id WHERE T2.tourney_name  =  'WTA Championships' INTERSECT SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id WHERE T2.tourney_name  =  'Australian Open'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name, players.country_code FROM players JOIN matches WHERE matches.tourney_name = 'terminal' INTERSECT SELECT players.first_name, players.country_code FROM players JOIN matches WHERE players.last_name = 'terminal'", "gold": "SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id WHERE T2.tourney_name  =  'WTA Championships' INTERSECT SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id WHERE T2.tourney_name  =  'Australian Open'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name, players.country_code FROM players JOIN matches ORDER BY matches.year Asc LIMIT 1", "gold": "SELECT first_name ,  country_code FROM players ORDER BY birth_date LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.country_code FROM players JOIN matches ORDER BY matches.year Asc LIMIT 1", "gold": "SELECT first_name ,  country_code FROM players ORDER BY birth_date LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.last_name FROM players ORDER BY players.birth_date Asc", "gold": "SELECT first_name ,  last_name FROM players ORDER BY birth_date", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name, players.last_name FROM players ORDER BY players.birth_date Asc", "gold": "SELECT first_name ,  last_name FROM players ORDER BY birth_date", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.first_name, players.last_name FROM players WHERE players.hand = 'terminal' ORDER BY players.birth_date Asc", "gold": "SELECT first_name ,  last_name FROM players WHERE hand  =  'L' ORDER BY birth_date", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.birth_date FROM players WHERE players.country_code = 'terminal' ORDER BY players.birth_date Asc", "gold": "SELECT first_name ,  last_name FROM players WHERE hand  =  'L' ORDER BY birth_date", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.country_code FROM players JOIN matches GROUP BY matches.loser_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id ORDER BY T2.tours DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT players.first_name, players.country_code FROM players JOIN matches GROUP BY matches.winner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.country_code ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id ORDER BY T2.tours DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 4}}}, {"predicted": "SELECT matches.year FROM matches GROUP BY matches.year ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT YEAR FROM matches GROUP BY YEAR ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT matches.year FROM matches GROUP BY matches.year ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT YEAR FROM matches GROUP BY YEAR ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.first_name, matches.score FROM players JOIN matches GROUP BY matches.winner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT winner_name ,  winner_rank_points FROM matches GROUP BY winner_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.first_name, matches.score FROM players JOIN matches GROUP BY matches.loser_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT winner_name ,  winner_rank_points FROM matches GROUP BY winner_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.first_name FROM matches JOIN players ORDER BY matches.score Desc LIMIT 1", "gold": "SELECT winner_name FROM matches WHERE tourney_name  =  'Australian Open' ORDER BY winner_rank_points DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT matches.winner_name FROM players JOIN matches WHERE players.last_name = 'terminal' ORDER BY matches.minutes Desc LIMIT 1", "gold": "SELECT winner_name FROM matches WHERE tourney_name  =  'Australian Open' ORDER BY winner_rank_points DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.first_name, matches.winner_name FROM matches JOIN players ORDER BY matches.minutes Desc LIMIT 1", "gold": "SELECT winner_name ,  loser_name FROM matches ORDER BY minutes DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.last_name FROM matches JOIN players ORDER BY matches.minutes Desc LIMIT 1", "gold": "SELECT winner_name ,  loser_name FROM matches ORDER BY minutes DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(matches.score), players.first_name FROM matches JOIN players GROUP BY players.first_name", "gold": "SELECT avg(ranking) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.first_name, Avg(matches.score) FROM rankings JOIN players GROUP BY rankings.player_id", "gold": "SELECT avg(ranking) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(matches.minutes), players.first_name FROM matches JOIN players GROUP BY players.first_name", "gold": "SELECT sum(ranking_points) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.first_name, rankings.ranking_points FROM rankings JOIN players", "gold": "SELECT sum(ranking_points) ,  T1.first_name FROM players AS T1 JOIN rankings AS T2 ON T1.player_id  =  T2.player_id GROUP BY T1.first_name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Count(*), players.country_code FROM players GROUP BY players.country_code", "gold": "SELECT count(*) ,  country_code FROM players GROUP BY country_code", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.country_code, Count(*) FROM players GROUP BY players.country_code", "gold": "SELECT count(*) ,  country_code FROM players GROUP BY country_code", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.country_code FROM players GROUP BY players.country_code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT country_code FROM players GROUP BY country_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.country_code FROM players GROUP BY players.country_code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT country_code FROM players GROUP BY country_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT players.country_code FROM players GROUP BY players.country_code HAVING Count(*) > 'terminal'", "gold": "SELECT country_code FROM players GROUP BY country_code HAVING count(*)  >  50", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT players.country_code FROM players GROUP BY players.country_code HAVING Count(*) > 'terminal'", "gold": "SELECT country_code FROM players GROUP BY country_code HAVING count(*)  >  50", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*), matches.score FROM matches GROUP BY matches.score", "gold": "SELECT sum(tours) ,  ranking_date FROM rankings GROUP BY ranking_date", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(rankings.tours), rankings.ranking FROM rankings GROUP BY rankings.ranking", "gold": "SELECT sum(tours) ,  ranking_date FROM rankings GROUP BY ranking_date", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), matches.year FROM matches GROUP BY matches.year", "gold": "SELECT count(*) ,  YEAR FROM matches GROUP BY YEAR", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT matches.year, Count(*) FROM matches GROUP BY matches.year", "gold": "SELECT count(*) ,  YEAR FROM matches GROUP BY YEAR", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT matches.winner_name, matches.score FROM matches ORDER BY matches.loser_age Asc LIMIT 1", "gold": "SELECT DISTINCT winner_name ,  winner_rank FROM matches ORDER BY winner_age LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, matches.score FROM players JOIN matches ORDER BY matches.loser_age Asc LIMIT 1", "gold": "SELECT DISTINCT winner_name ,  winner_rank FROM matches ORDER BY winner_age LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(DISTINCT players.player_id) FROM players WHERE players.first_name = 'terminal'", "gold": "SELECT count(DISTINCT winner_name) FROM matches WHERE tourney_name  =  'WTA Championships' AND winner_hand  =  'L'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM players JOIN matches WHERE players.country_code = 'terminal'", "gold": "SELECT count(DISTINCT winner_name) FROM matches WHERE tourney_name  =  'WTA Championships' AND winner_hand  =  'L'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT matches.winner_name, players.country_code, players.birth_date FROM matches JOIN players ORDER BY matches.score Desc LIMIT 1", "gold": "SELECT T1.first_name ,  T1.country_code ,  T1.birth_date FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id ORDER BY T2.winner_rank_points DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT players.first_name, players.country_code, matches.minutes FROM players JOIN matches ORDER BY matches.minutes Desc LIMIT 1", "gold": "SELECT T1.first_name ,  T1.country_code ,  T1.birth_date FROM players AS T1 JOIN matches AS T2 ON T1.player_id  =  T2.winner_id ORDER BY T2.winner_rank_points DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*), players.country_code FROM players GROUP BY players.country_code", "gold": "SELECT count(*) ,  hand FROM players GROUP BY hand", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT players.country_code, Count(*) FROM players GROUP BY players.country_code", "gold": "SELECT count(*) ,  hand FROM players GROUP BY hand", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM ship", "gold": "SELECT count(*) FROM ship WHERE disposition_of_ship  =  'Captured'", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT ship.name, ship.tonnage FROM ship ORDER BY ship.name Desc", "gold": "SELECT name ,  tonnage FROM ship ORDER BY name DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT battle.name, battle.date, battle.result FROM battle", "gold": "SELECT name ,  date FROM battle", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Max(death.killed), Min(death.killed), death.note FROM death GROUP BY death.note", "gold": "SELECT max(killed) ,  min(killed) FROM death", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Avg(death.injured), death.caused_by_ship_id FROM death GROUP BY death.caused_by_ship_id", "gold": "SELECT avg(injured) FROM death", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT battle.name, ship.disposition_of_ship FROM ship JOIN battle WHERE ship.tonnage = 'terminal'", "gold": "SELECT T1.killed ,  T1.injured FROM death AS T1 JOIN ship AS t2 ON T1.caused_by_ship_id  =  T2.id WHERE T2.tonnage  =  't'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT battle.name, battle.result FROM battle WHERE battle.bulgarian_commander != 'terminal'", "gold": "SELECT name ,  RESULT FROM battle WHERE bulgarian_commander != 'Boril'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT battle.id, battle.name FROM battle JOIN ship WHERE ship.ship_type = 'terminal'", "gold": "SELECT DISTINCT T1.id ,  T1.name FROM battle AS T1 JOIN ship AS T2 ON T1.id  =  T2.lost_in_battle WHERE T2.ship_type  =  'Brig'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT ship.id, battle.name FROM death JOIN battle JOIN ship WHERE death.killed > 'terminal'", "gold": "SELECT T1.id ,  T1.name FROM battle AS T1 JOIN ship AS T2 ON T1.id  =  T2.lost_in_battle JOIN death AS T3 ON T2.id  =  T3.caused_by_ship_id GROUP BY T1.id HAVING sum(T3.killed)  >  10", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT ship.id, ship.name FROM ship JOIN death GROUP BY ship.id ORDER BY death.injured Desc LIMIT 1", "gold": "SELECT T2.id ,  T2.name FROM death AS T1 JOIN ship AS t2 ON T1.caused_by_ship_id  =  T2.id GROUP BY T2.id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT DISTINCT battle.name FROM battle WHERE battle.bulgarian_commander = 'terminal' AND battle.latin_commander = 'terminal'", "gold": "SELECT name FROM battle WHERE bulgarian_commander  =  'Kaloyan' AND latin_commander  =  'Baldwin I'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT battle.result) FROM battle", "gold": "SELECT count(DISTINCT RESULT) FROM battle", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM ship WHERE ship.tonnage != 'terminal'", "gold": "SELECT count(*) FROM battle WHERE id NOT IN ( SELECT lost_in_battle FROM ship WHERE tonnage  =  '225' );", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT battle.name, battle.date FROM ship JOIN battle WHERE ship.name = 'terminal' INTERSECT SELECT battle.name, battle.date FROM ship JOIN battle WHERE ship.name = 'terminal'", "gold": "SELECT T1.name ,  T1.date FROM battle AS T1 JOIN ship AS T2 ON T1.id  =  T2.lost_in_battle WHERE T2.name  =  'Lettice' INTERSECT SELECT T1.name ,  T1.date FROM battle AS T1 JOIN ship AS T2 ON T1.id  =  T2.lost_in_battle WHERE T2.name  =  'HMS Atalanta'", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT battle.name, battle.result, battle.bulgarian_commander FROM battle WHERE battle.bulgarian_commander != 'terminal'", "gold": "SELECT name ,  RESULT ,  bulgarian_commander FROM battle EXCEPT SELECT T1.name ,  T1.result ,  T1.bulgarian_commander FROM battle AS T1 JOIN ship AS T2 ON T1.id  =  T2.lost_in_battle WHERE T2.location  =  'English Channel'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT death.killed FROM death WHERE death.killed LIKE 'terminal'", "gold": "SELECT note FROM death WHERE note LIKE '%East%'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Addresses.address_id, Addresses.zip_postcode FROM Addresses", "gold": "SELECT line_1 ,  line_2 FROM addresses", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Addresses.line_1, Addresses.line_3 FROM Addresses", "gold": "SELECT line_1 ,  line_2 FROM addresses", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Courses", "gold": "SELECT count(*) FROM Courses", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Courses", "gold": "SELECT count(*) FROM Courses", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_description FROM Courses WHERE Courses.course_name = 'terminal'", "gold": "SELECT course_description FROM Courses WHERE course_name  =  'math'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Courses.course_description FROM Courses WHERE Courses.course_name = 'terminal'", "gold": "SELECT course_description FROM Courses WHERE course_name  =  'math'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Addresses.zip_postcode FROM Addresses WHERE Addresses.city = 'terminal'", "gold": "SELECT zip_postcode FROM Addresses WHERE city  =  'Port Chelsea'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Addresses.zip_postcode FROM Addresses WHERE Addresses.city = 'terminal'", "gold": "SELECT zip_postcode FROM Addresses WHERE city  =  'Port Chelsea'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Departments.department_name, Departments.department_id FROM Departments JOIN Courses GROUP BY Departments.department_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.department_name ,  T1.department_id FROM Degree_Programs AS T1 JOIN Departments AS T2 ON T1.department_id  =  T2.department_id GROUP BY T1.department_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Departments.department_name, Degree_Programs.department_id FROM Departments JOIN Degree_Programs GROUP BY Degree_Programs.department_id ORDER BY Count(*) Desc LIMIT 1", "gold": "select t2.department_name ,  t1.department_id from degree_programs as t1 join departments as t2 on t1.department_id  =  t2.department_id group by t1.department_id order by count(*) desc limit 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(DISTINCT Degree_Programs.department_id) FROM Degree_Programs", "gold": "SELECT count(DISTINCT department_id) FROM Degree_Programs", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Departments.department_name) FROM Departments", "gold": "SELECT count(DISTINCT department_id) FROM Degree_Programs", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Degree_Programs.degree_summary_name) FROM Degree_Programs", "gold": "SELECT count(DISTINCT degree_summary_name) FROM Degree_Programs", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Courses.course_name) FROM Courses", "gold": "SELECT count(DISTINCT degree_summary_name) FROM Degree_Programs", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Courses JOIN Departments WHERE Departments.department_name = 'terminal'", "gold": "SELECT count(*) FROM Departments AS T1 JOIN Degree_Programs AS T2 ON T1.department_id  =  T2.department_id WHERE T1.department_name  =  'engineer'", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Departments JOIN Degree_Programs WHERE Departments.department_name = 'terminal'", "gold": "SELECT count(*) FROM Departments AS T1 JOIN Degree_Programs AS T2 ON T1.department_id  =  T2.department_id WHERE T1.department_name  =  'engineer'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Courses.course_name, Courses.course_description FROM Courses", "gold": "SELECT section_name ,  section_description FROM Sections", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_name, Courses.course_description FROM Courses", "gold": "SELECT section_name ,  section_description FROM Sections", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_name, Courses.course_id FROM Courses JOIN Sections GROUP BY Courses.course_id HAVING Count(*) <= 'terminal'", "gold": "SELECT T1.course_name ,  T1.course_id FROM Courses AS T1 JOIN Sections AS T2 ON T1.course_id  =  T2.course_id GROUP BY T1.course_id HAVING count(*)  <=  2", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Courses.course_name, Courses.course_id FROM Courses JOIN Sections GROUP BY Courses.course_id HAVING Count(*) < 'terminal'", "gold": "SELECT T1.course_name ,  T1.course_id FROM Courses AS T1 JOIN Sections AS T2 ON T1.course_id  =  T2.course_id GROUP BY T1.course_id HAVING count(*)  <=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Sections.section_name FROM Sections WHERE Sections.section_name = 'terminal' ORDER BY Sections.section_name Asc", "gold": "SELECT section_name FROM Sections ORDER BY section_name DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Departments.department_name FROM Departments ORDER BY Departments.department_name Asc", "gold": "SELECT section_name FROM Sections ORDER BY section_name DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Students.first_name, Student_Enrolment.degree_program_id FROM Student_Enrolment JOIN Students GROUP BY Student_Enrolment.degree_program_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.semester_name ,  T1.semester_id FROM Semesters AS T1 JOIN Student_Enrolment AS T2 ON T1.semester_id  =  T2.semester_id GROUP BY T1.semester_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Courses.course_name, Student_Enrolment.degree_program_id FROM Courses JOIN Student_Enrolment GROUP BY Student_Enrolment.degree_program_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.semester_name ,  T1.semester_id FROM Semesters AS T1 JOIN Student_Enrolment AS T2 ON T1.semester_id  =  T2.semester_id GROUP BY T1.semester_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Departments.department_description FROM Departments WHERE Departments.department_name LIKE 'terminal'", "gold": "SELECT department_description FROM Departments WHERE department_name LIKE '%computer%'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Departments.department_description FROM Departments WHERE Departments.department_name LIKE 'terminal'", "gold": "SELECT department_description FROM Departments WHERE department_name LIKE '%computer%'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Student_Enrolment.student_id FROM Students JOIN Student_Enrolment GROUP BY Student_Enrolment.student_id", "gold": "SELECT T1.first_name ,  T1.middle_name ,  T1.last_name ,  T1.student_id FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id GROUP BY T1.student_id HAVING count(*)  =  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Student_Enrolment.student_id FROM Students JOIN Student_Enrolment GROUP BY Student_Enrolment.student_id", "gold": "SELECT T1.first_name ,  T1.middle_name ,  T1.last_name ,  T1.student_id FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id GROUP BY T1.student_id HAVING count(*)  =  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Courses JOIN Students WHERE Courses.course_name = 'terminal'", "gold": "SELECT DISTINCT T1.first_name ,  T1.middle_name ,  T1.last_name FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id JOIN Degree_Programs AS T3 ON T2.degree_program_id  =  T3.degree_program_id WHERE T3.degree_summary_name  =  'Bachelor'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Students JOIN Student_Enrolment WHERE Students.ssn = (SELECT Max(Students.ssn) FROM Students)", "gold": "SELECT DISTINCT T1.first_name ,  T1.middle_name ,  T1.last_name FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id JOIN Degree_Programs AS T3 ON T2.degree_program_id  =  T3.degree_program_id WHERE T3.degree_summary_name  =  'Bachelor'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Degree_Programs.degree_summary_name FROM Student_Enrolment JOIN Degree_Programs GROUP BY Student_Enrolment.degree_program_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.degree_summary_name FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id GROUP BY T1.degree_summary_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Courses.course_name FROM Courses JOIN Student_Enrolment_Courses GROUP BY Courses.course_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.degree_summary_name FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id GROUP BY T1.degree_summary_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Courses.course_id, Courses.course_description FROM Courses JOIN Student_Enrolment_Courses GROUP BY Courses.course_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.degree_program_id ,  T1.degree_summary_name FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id GROUP BY T1.degree_program_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Student_Enrolment_Courses.course_id, Courses.course_name FROM Courses JOIN Student_Enrolment_Courses GROUP BY Student_Enrolment_Courses.course_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.degree_program_id ,  T1.degree_summary_name FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id GROUP BY T1.degree_program_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Student_Enrolment.student_enrolment_id, Students.middle_name, Students.middle_name, Student_Enrolment.student_enrolment_id FROM Students JOIN Student_Enrolment GROUP BY Student_Enrolment.student_enrolment_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.student_id ,  T1.first_name ,  T1.middle_name ,  T1.last_name ,  count(*) ,  T1.student_id FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 6, "pred_total": 4}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 6, "pred_total": 4}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Student_Enrolment.student_id FROM Students JOIN Student_Enrolment GROUP BY Students.student_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.student_id ,  T1.first_name ,  T1.middle_name ,  T1.last_name ,  count(*) ,  T1.student_id FROM Students AS T1 JOIN Student_Enrolment AS T2 ON T1.student_id  =  T2.student_id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 6, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 6, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Courses.course_name FROM Courses WHERE Departments.department_id NOT IN (SELECT Student_Enrolment.degree_program_id FROM Student_Enrolment)", "gold": "SELECT semester_name FROM Semesters WHERE semester_id NOT IN( SELECT semester_id FROM Student_Enrolment )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Courses.course_name FROM Semesters EXCEPT SELECT Student_Enrolment.semester_id FROM Student_Enrolment", "gold": "SELECT semester_name FROM Semesters WHERE semester_id NOT IN( SELECT semester_id FROM Student_Enrolment )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Courses.course_name FROM Courses JOIN Student_Enrolment_Courses", "gold": "SELECT DISTINCT T1.course_name FROM Courses AS T1 JOIN Student_Enrolment_Courses AS T2 ON T1.course_id  =  T2.course_id", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_name FROM Courses JOIN Student_Enrolment_Courses", "gold": "SELECT DISTINCT T1.course_name FROM Courses AS T1 JOIN Student_Enrolment_Courses AS T2 ON T1.course_id  =  T2.course_id", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_name FROM Courses JOIN Student_Enrolment_Courses GROUP BY Courses.course_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT  T1.course_name FROM Courses AS T1 JOIN Student_Enrolment_Courses AS T2 ON T1.course_id  =  T2.course_id GROUP BY T1.course_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Courses.course_name FROM Courses JOIN Student_Enrolment_Courses GROUP BY Courses.course_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT  T1.course_name FROM Courses AS T1 JOIN Student_Enrolment_Courses AS T2 ON T1.course_id  =  T2.course_id GROUP BY T1.course_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Students.last_name FROM Addresses JOIN Students WHERE Addresses.state_province_county = 'terminal' EXCEPT SELECT Students.last_name FROM Addresses JOIN Students WHERE Addresses.state_province_county = 'terminal'", "gold": "SELECT T1.last_name FROM Students AS T1 JOIN Addresses AS T2 ON T1.current_address_id  =  T2.address_id WHERE T2.state_province_county  =  'NorthCarolina' EXCEPT SELECT DISTINCT T3.last_name FROM Students AS T3 JOIN Student_Enrolment AS T4 ON T3.student_id  =  T4.student_id", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Students.last_name FROM Students EXCEPT SELECT Students.last_name FROM Addresses JOIN Students WHERE Addresses.state_province_county = 'terminal'", "gold": "SELECT T1.last_name FROM Students AS T1 JOIN Addresses AS T2 ON T1.current_address_id  =  T2.address_id WHERE T2.state_province_county  =  'NorthCarolina' EXCEPT SELECT DISTINCT T3.last_name FROM Students AS T3 JOIN Student_Enrolment AS T4 ON T3.student_id  =  T4.student_id", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Transcripts.transcript_date, Courses.course_id FROM Courses JOIN Transcripts GROUP BY Courses.course_name HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.transcript_date ,  T1.transcript_id FROM Transcript_Contents AS T1 JOIN Transcripts AS T2 ON T1.transcript_id  =  T2.transcript_id GROUP BY T1.transcript_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Transcripts.transcript_date, Courses.course_id FROM Courses JOIN Transcripts GROUP BY Courses.course_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.transcript_date ,  T1.transcript_id FROM Transcript_Contents AS T1 JOIN Transcripts AS T2 ON T1.transcript_id  =  T2.transcript_id GROUP BY T1.transcript_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Students.cell_mobile_number FROM Students WHERE Students.first_name = 'terminal' AND Students.last_name = 'terminal'", "gold": "SELECT cell_mobile_number FROM Students WHERE first_name  =  'Timmothy' AND last_name  =  'Ward'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Students.cell_mobile_number FROM Students WHERE Students.first_name = 'terminal' AND Students.last_name = 'terminal'", "gold": "select cell_mobile_number from students where first_name  =  'timmothy' and last_name  =  'ward'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Students ORDER BY Students.ssn Asc LIMIT 1", "gold": "SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_first_registered ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Students ORDER BY Students.date_first_registered Asc LIMIT 1", "gold": "SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_first_registered ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Students ORDER BY Students.date_left Asc LIMIT 1", "gold": "SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_left ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Students.first_name, Students.middle_name, Students.last_name FROM Students ORDER BY Students.date_left Asc LIMIT 1", "gold": "SELECT first_name ,  middle_name ,  last_name FROM Students ORDER BY date_left ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT DISTINCT Students.first_name FROM Students", "gold": "SELECT first_name FROM Students WHERE current_address_id != permanent_address_id", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Students.first_name FROM Students WHERE Students.email_address = 'terminal'", "gold": "SELECT first_name FROM Students WHERE current_address_id != permanent_address_id", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Addresses.address_id, Students.email_address FROM Addresses JOIN Students GROUP BY Addresses.address_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.address_id ,  T1.line_1 ,  T1.line_2 FROM Addresses AS T1 JOIN Students AS T2 ON T1.address_id  =  T2.current_address_id GROUP BY T1.address_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Addresses.address_id, Addresses.line_2, Addresses.city FROM Addresses JOIN Students GROUP BY Students.current_address_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.address_id ,  T1.line_1 ,  T1.line_2 FROM Addresses AS T1 JOIN Students AS T2 ON T1.address_id  =  T2.current_address_id GROUP BY T1.address_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Avg(Transcripts.transcript_date) FROM Transcripts", "gold": "SELECT avg(transcript_date) FROM Transcripts", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(Transcripts.transcript_date) FROM Transcripts", "gold": "SELECT avg(transcript_date) FROM Transcripts", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Transcripts.transcript_date, Transcripts.other_details FROM Transcripts ORDER BY Transcripts.transcript_date Asc LIMIT 1", "gold": "SELECT transcript_date ,  other_details FROM Transcripts ORDER BY transcript_date ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Transcripts.transcript_date, Transcripts.other_details FROM Transcripts", "gold": "SELECT transcript_date ,  other_details FROM Transcripts ORDER BY transcript_date ASC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Transcripts", "gold": "SELECT count(*) FROM Transcripts", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Transcripts", "gold": "SELECT count(*) FROM Transcripts", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Transcripts.transcript_date FROM Transcripts ORDER BY Transcripts.transcript_date Desc LIMIT 1", "gold": "SELECT transcript_date FROM Transcripts ORDER BY transcript_date DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Students.date_first_registered FROM Students ORDER BY Students.date_first_registered Desc LIMIT 1", "gold": "SELECT transcript_date FROM Transcripts ORDER BY transcript_date DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*), Student_Enrolment.degree_program_id FROM Student_Enrolment JOIN Student_Enrolment_Courses GROUP BY Student_Enrolment_Courses.course_id", "gold": "SELECT count(*) ,  student_course_id FROM Transcript_Contents GROUP BY student_course_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Student_Enrolment_Courses.course_id FROM Courses JOIN Student_Enrolment_Courses GROUP BY Student_Enrolment_Courses.course_id", "gold": "SELECT count(*) ,  student_course_id FROM Transcript_Contents GROUP BY student_course_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT Transcripts.transcript_date, Transcripts.transcript_id FROM Transcripts GROUP BY Transcripts.transcript_date ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T2.transcript_date ,  T1.transcript_id FROM Transcript_Contents AS T1 JOIN Transcripts AS T2 ON T1.transcript_id  =  T2.transcript_id GROUP BY T1.transcript_id ORDER BY count(*) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Transcripts.transcript_date, Transcripts.transcript_id FROM Transcripts JOIN Transcript_Contents GROUP BY Transcript_Contents.transcript_id ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT T2.transcript_date ,  T1.transcript_id FROM Transcript_Contents AS T1 JOIN Transcripts AS T2 ON T1.transcript_id  =  T2.transcript_id GROUP BY T1.transcript_id ORDER BY count(*) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Students.date_left FROM Courses JOIN Students WHERE Student_Enrolment_Courses.course_id = 'terminal' INTERSECT SELECT Students.date_left FROM Courses JOIN Student_Enrolment_Courses WHERE Student_Enrolment_Courses.course_id = 'terminal'", "gold": "SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Master' INTERSECT SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Bachelor'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Courses.course_id FROM Courses JOIN Student_Enrolment_Courses WHERE Courses.course_name = 'terminal' INTERSECT SELECT Student_Enrolment_Courses.course_id FROM Student_Enrolment_Courses WHERE Student_Enrolment_Courses.student_enrolment_id = 'terminal'", "gold": "SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Master' INTERSECT SELECT DISTINCT T2.semester_id FROM Degree_Programs AS T1 JOIN Student_Enrolment AS T2 ON T1.degree_program_id  =  T2.degree_program_id WHERE degree_summary_name  =  'Bachelor'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(DISTINCT Students.email_address) FROM Students", "gold": "SELECT count(DISTINCT current_address_id) FROM Students", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT Addresses.zip_postcode FROM Addresses JOIN Students", "gold": "SELECT count(DISTINCT current_address_id) FROM Students", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Student_Enrolment.other_details FROM Student_Enrolment WHERE Student_Enrolment.other_details = 'terminal' ORDER BY Student_Enrolment.other_details Asc", "gold": "SELECT other_student_details FROM Students ORDER BY other_student_details DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Students.other_student_details FROM Students ORDER BY Students.other_student_details Asc", "gold": "SELECT other_student_details FROM Students ORDER BY other_student_details DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Courses.course_description FROM Courses", "gold": "SELECT section_description FROM Sections WHERE section_name  =  'h'", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Courses.course_description FROM Courses WHERE Courses.course_name = 'terminal'", "gold": "SELECT section_description FROM Sections WHERE section_name  =  'h'", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Students.first_name FROM Students JOIN Addresses WHERE Addresses.country = 'terminal' OR Students.cell_mobile_number = 'terminal'", "gold": "select t1.first_name from students as t1 join addresses as t2 on t1.permanent_address_id  =  t2.address_id where t2.country  =  'haiti' or t1.cell_mobile_number  =  '09700166582'", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Students.first_name FROM Addresses JOIN Students WHERE Addresses.country = 'terminal' OR Students.cell_mobile_number = 'terminal'", "gold": "select t1.first_name from students as t1 join addresses as t2 on t1.permanent_address_id  =  t2.address_id where t2.country  =  'haiti' or t1.cell_mobile_number  =  '09700166582'", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon ORDER BY Cartoon.Title Asc", "gold": "SELECT Title FROM Cartoon ORDER BY title", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon ORDER BY Cartoon.Title Asc", "gold": "SELECT Title FROM Cartoon ORDER BY title", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT Title FROM Cartoon WHERE Directed_by = \"Ben Jones\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT Title FROM Cartoon WHERE Directed_by = \"Ben Jones\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Cartoon WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT count(*) FROM Cartoon WHERE Written_by = \"Joseph Kuhr\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Cartoon WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT count(*) FROM Cartoon WHERE Written_by = \"Joseph Kuhr\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Cartoon.Title, Cartoon.Directed_by FROM Cartoon ORDER BY Cartoon.Original_air_date Asc", "gold": "SELECT title ,  Directed_by FROM Cartoon ORDER BY Original_air_date", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Cartoon.Title, Cartoon.Directed_by FROM Cartoon JOIN TV_series", "gold": "SELECT title ,  Directed_by FROM Cartoon ORDER BY Original_air_date", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon WHERE Cartoon.Directed_by = 'terminal' OR Cartoon.Directed_by = 'terminal'", "gold": "SELECT Title FROM Cartoon WHERE Directed_by = \"Ben Jones\" OR Directed_by = \"Brandon Vietti\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon WHERE Cartoon.Directed_by = 'terminal' OR Cartoon.Directed_by = 'terminal'", "gold": "SELECT Title FROM Cartoon WHERE Directed_by = \"Ben Jones\" OR Directed_by = \"Brandon Vietti\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT TV_Channel.Country, Count(*) FROM TV_Channel GROUP BY TV_Channel.Country ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Country ,  count(*) FROM TV_Channel GROUP BY Country ORDER BY count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT TV_Channel.Country, Count(*) FROM TV_Channel GROUP BY TV_Channel.Country ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Country ,  count(*) FROM TV_Channel GROUP BY Country ORDER BY count(*) DESC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(DISTINCT TV_Channel.Content), TV_Channel.Content FROM TV_Channel", "gold": "SELECT count(DISTINCT series_name) ,  count(DISTINCT content) FROM TV_Channel;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT TV_Channel.Content), Count(DISTINCT TV_Channel.Content) FROM TV_Channel", "gold": "SELECT count(DISTINCT series_name) ,  count(DISTINCT content) FROM TV_Channel;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT TV_Channel.Content FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT Content FROM TV_Channel WHERE series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Content FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT Content FROM TV_Channel WHERE series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Package_Option FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT Package_Option FROM TV_Channel WHERE series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Hight_definition_TV FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT Package_Option FROM TV_Channel WHERE series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT count(*) FROM TV_Channel WHERE LANGUAGE = \"English\";", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM TV_Channel WHERE TV_Channel.Language = 'terminal'", "gold": "SELECT count(*) FROM TV_Channel WHERE LANGUAGE = \"English\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Language, Count(*) FROM TV_Channel JOIN TV_series GROUP BY TV_Channel.Language ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT LANGUAGE ,  count(*) FROM TV_Channel GROUP BY LANGUAGE ORDER BY count(*) ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT TV_Channel.Language, Count(*) FROM TV_Channel GROUP BY TV_Channel.Language ORDER BY Count(*) Asc LIMIT 1", "gold": "SELECT LANGUAGE ,  count(*) FROM TV_Channel GROUP BY LANGUAGE ORDER BY count(*) ASC LIMIT 1;", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT TV_Channel.Language, Count(*) FROM TV_Channel GROUP BY TV_Channel.Language", "gold": "SELECT LANGUAGE ,  count(*) FROM TV_Channel GROUP BY LANGUAGE", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Language, Count(*) FROM TV_Channel GROUP BY TV_Channel.Language", "gold": "SELECT LANGUAGE ,  count(*) FROM TV_Channel GROUP BY LANGUAGE", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.series_name FROM TV_Channel JOIN Cartoon WHERE Cartoon.Title = 'terminal'", "gold": "SELECT T1.series_name FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T2.Title = \"The Rise of the Blue Beetle!\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Content FROM Cartoon JOIN TV_Channel WHERE Cartoon.Title = 'terminal'", "gold": "SELECT T1.series_name FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T2.Title = \"The Rise of the Blue Beetle!\";", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon JOIN TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT T2.Title FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T1.series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Cartoon.Title FROM Cartoon JOIN TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT T2.Title FROM TV_Channel AS T1 JOIN Cartoon AS T2 ON T1.id = T2.Channel WHERE T1.series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_series.Rating FROM TV_series ORDER BY TV_series.Rating Asc", "gold": "SELECT Episode FROM TV_series ORDER BY rating", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT TV_series.Rating FROM TV_series ORDER BY TV_series.Rating Asc", "gold": "SELECT Episode FROM TV_series ORDER BY rating", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT TV_series.Channel, TV_series.Rating FROM TV_series ORDER BY TV_series.Rating Desc LIMIT 1", "gold": "SELECT Episode ,  Rating FROM TV_series ORDER BY Rating DESC LIMIT 3;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT TV_series.Weekly_Rank, TV_series.Share FROM TV_series ORDER BY TV_series.Rating Desc LIMIT 1", "gold": "SELECT Episode ,  Rating FROM TV_series ORDER BY Rating DESC LIMIT 3;", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Min(TV_series.Share), Max(TV_series.Share) FROM TV_series", "gold": "SELECT max(SHARE) , min(SHARE) FROM TV_series;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Max(TV_series.Share), Min(TV_series.Share) FROM TV_series", "gold": "SELECT max(SHARE) , min(SHARE) FROM TV_series;", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT TV_series.Air_Date FROM TV_series JOIN TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT Air_Date FROM TV_series WHERE Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_series.Air_Date FROM TV_series WHERE TV_series.Episode = 'terminal'", "gold": "SELECT Air_Date FROM TV_series WHERE Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_series.Rating FROM TV_series WHERE TV_series.Episode = 'terminal'", "gold": "SELECT Weekly_Rank FROM TV_series WHERE Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_series.Rating FROM TV_series WHERE TV_series.Episode = 'terminal'", "gold": "SELECT Weekly_Rank FROM TV_series WHERE Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Language FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT T1.series_name FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T2.Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.series_name FROM TV_series JOIN TV_Channel WHERE TV_series.Episode = 'terminal'", "gold": "SELECT T1.series_name FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T2.Episode = \"A Love of a Lifetime\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_series.Episode FROM TV_Channel JOIN TV_series WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT T2.Episode FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T1.series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Hight_definition_TV FROM TV_Channel WHERE TV_Channel.series_name = 'terminal'", "gold": "SELECT T2.Episode FROM TV_Channel AS T1 JOIN TV_series AS T2 ON T1.id = T2.Channel WHERE T1.series_name = \"Sky Radio\";", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Cartoon.Directed_by FROM Cartoon GROUP BY Cartoon.Directed_by", "gold": "SELECT count(*) ,  Directed_by FROM cartoon GROUP BY Directed_by", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Cartoon.Directed_by FROM Cartoon GROUP BY Cartoon.Directed_by", "gold": "SELECT count(*) ,  Directed_by FROM cartoon GROUP BY Directed_by", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Cartoon.Production_code, Cartoon.Channel FROM Cartoon ORDER BY Cartoon.Original_air_date Desc LIMIT 1", "gold": "select production_code ,  channel from cartoon order by original_air_date desc limit 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Cartoon.Production_code, Cartoon.Channel FROM Cartoon ORDER BY Cartoon.Original_air_date Desc LIMIT 1", "gold": "select production_code ,  channel from cartoon order by original_air_date desc limit 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT TV_Channel.Country, TV_Channel.series_name FROM TV_Channel WHERE TV_Channel.Hight_definition_TV = 'terminal'", "gold": "SELECT package_option ,  series_name FROM TV_Channel WHERE hight_definition_TV  =  \"yes\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.series_name, TV_Channel.series_name FROM TV_Channel WHERE TV_Channel.Hight_definition_TV = 'terminal'", "gold": "SELECT package_option ,  series_name FROM TV_Channel WHERE hight_definition_TV  =  \"yes\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Country FROM Cartoon JOIN TV_Channel WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Country FROM Cartoon JOIN TV_Channel WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Hight_definition_TV FROM TV_Channel EXCEPT SELECT TV_Channel.Hight_definition_TV FROM TV_Channel JOIN Cartoon WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT country FROM TV_Channel EXCEPT SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Country FROM TV_Channel EXCEPT SELECT TV_Channel.Country FROM TV_Channel JOIN Cartoon WHERE Cartoon.Written_by = 'terminal'", "gold": "SELECT country FROM TV_Channel EXCEPT SELECT T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.written_by  =  'Todd Casey'", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.series_name, TV_Channel.Country FROM TV_Channel JOIN Cartoon WHERE Cartoon.Directed_by = 'terminal' AND Cartoon.Directed_by = 'terminal'", "gold": "SELECT T1.series_name ,  T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.directed_by  =  'Michael Chang' INTERSECT SELECT T1.series_name ,  T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.directed_by  =  'Ben Jones'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.series_name, TV_Channel.Country FROM TV_Channel JOIN Cartoon WHERE Cartoon.Directed_by = 'terminal' AND Cartoon.Directed_by = 'terminal'", "gold": "SELECT T1.series_name ,  T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.directed_by  =  'Michael Chang' INTERSECT SELECT T1.series_name ,  T1.country FROM TV_Channel AS T1 JOIN cartoon AS T2 ON T1.id = T2.Channel WHERE T2.directed_by  =  'Ben Jones'", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Pixel_aspect_ratio_PAR, TV_Channel.Country FROM TV_Channel WHERE TV_Channel.Language != 'terminal'", "gold": "SELECT Pixel_aspect_ratio_PAR ,  country FROM tv_channel WHERE LANGUAGE != 'English'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Pixel_aspect_ratio_PAR, TV_Channel.Country FROM TV_Channel WHERE TV_Channel.Language != 'terminal'", "gold": "SELECT Pixel_aspect_ratio_PAR ,  country FROM tv_channel WHERE LANGUAGE != 'English'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.id FROM TV_Channel GROUP BY TV_Channel.id HAVING Count(*) > 'terminal'", "gold": "SELECT id FROM tv_channel GROUP BY country HAVING count(*)  >  2", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT TV_Channel.id FROM TV_Channel GROUP BY TV_Channel.id HAVING Count(*) > 'terminal'", "gold": "SELECT id FROM tv_channel GROUP BY country HAVING count(*)  >  2", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT TV_Channel.id FROM TV_Channel EXCEPT SELECT Cartoon.Channel FROM Cartoon WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT id FROM TV_Channel EXCEPT SELECT channel FROM cartoon WHERE directed_by  =  'Ben Jones'", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.id FROM TV_Channel EXCEPT SELECT Cartoon.Channel FROM Cartoon WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT id FROM TV_Channel EXCEPT SELECT channel FROM cartoon WHERE directed_by  =  'Ben Jones'", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Hight_definition_TV FROM TV_Channel EXCEPT SELECT TV_Channel.Hight_definition_TV FROM Cartoon JOIN TV_Channel WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT package_option FROM TV_Channel WHERE id NOT IN (SELECT channel FROM cartoon WHERE directed_by  =  'Ben Jones')", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT TV_Channel.Hight_definition_TV FROM TV_Channel EXCEPT SELECT TV_Channel.Hight_definition_TV FROM Cartoon JOIN TV_Channel WHERE Cartoon.Directed_by = 'terminal'", "gold": "SELECT package_option FROM TV_Channel WHERE id NOT IN (SELECT channel FROM cartoon WHERE directed_by  =  'Ben Jones')", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM poker_player", "gold": "SELECT count(*) FROM poker_player", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM poker_player", "gold": "SELECT count(*) FROM poker_player", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player ORDER BY poker_player.Earnings Desc", "gold": "SELECT Earnings FROM poker_player ORDER BY Earnings DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player ORDER BY poker_player.Best_Finish Desc", "gold": "SELECT Earnings FROM poker_player ORDER BY Earnings DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT poker_player.Final_Table_Made, poker_player.Final_Table_Made FROM poker_player", "gold": "SELECT Final_Table_Made ,  Best_Finish FROM poker_player", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT poker_player.Final_Table_Made, poker_player.Final_Table_Made FROM poker_player", "gold": "SELECT Final_Table_Made ,  Best_Finish FROM poker_player", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(poker_player.Money_Rank) FROM poker_player", "gold": "SELECT avg(Earnings) FROM poker_player", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(poker_player.Money_Rank) FROM poker_player", "gold": "SELECT avg(Earnings) FROM poker_player", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player ORDER BY poker_player.Earnings Desc LIMIT 1", "gold": "SELECT Money_Rank FROM poker_player ORDER BY Earnings DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player ORDER BY poker_player.Earnings Desc LIMIT 1", "gold": "SELECT Money_Rank FROM poker_player ORDER BY Earnings DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Max(poker_player.Final_Table_Made) FROM poker_player WHERE poker_player.Best_Finish < 'terminal'", "gold": "SELECT max(Final_Table_Made) FROM poker_player WHERE Earnings  <  200000", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Max(poker_player.Final_Table_Made) FROM poker_player WHERE poker_player.Best_Finish < 'terminal'", "gold": "SELECT max(Final_Table_Made) FROM poker_player WHERE Earnings  <  200000", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people WHERE poker_player.Best_Finish > 'terminal'", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T2.Earnings  >  300000", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people WHERE poker_player.Best_Finish > 'terminal'", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T2.Earnings  >  300000", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people ORDER BY poker_player.Earnings Asc", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Final_Table_Made", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people ORDER BY poker_player.Best_Finish Asc", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Final_Table_Made", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Nationality FROM poker_player JOIN people ORDER BY poker_player.Earnings Asc LIMIT 1", "gold": "SELECT T1.Birth_Date FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Earnings ASC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT people.Nationality FROM poker_player JOIN people ORDER BY poker_player.Earnings Asc LIMIT 1", "gold": "SELECT T1.Birth_Date FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Earnings ASC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player JOIN people ORDER BY people.Height Desc LIMIT 1", "gold": "SELECT T2.Money_Rank FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T1.Height DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT poker_player.Money_Rank FROM poker_player JOIN people ORDER BY people.Height Desc LIMIT 1", "gold": "SELECT T2.Money_Rank FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T1.Height DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(poker_player.Money_Rank) FROM poker_player JOIN people WHERE people.Height > 'terminal'", "gold": "SELECT avg(T2.Earnings) FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T1.Height  >  200", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(poker_player.Money_Rank) FROM poker_player JOIN people WHERE people.Height > 'terminal'", "gold": "SELECT avg(T2.Earnings) FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID WHERE T1.Height  >  200", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people ORDER BY poker_player.Earnings Desc", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Earnings DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Name FROM poker_player JOIN people ORDER BY poker_player.Earnings Desc", "gold": "SELECT T1.Name FROM people AS T1 JOIN poker_player AS T2 ON T1.People_ID  =  T2.People_ID ORDER BY T2.Earnings DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Nationality, Count(*) FROM people GROUP BY people.Nationality", "gold": "SELECT Nationality ,  COUNT(*) FROM people GROUP BY Nationality", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Nationality, Count(*) FROM people GROUP BY people.Nationality", "gold": "SELECT Nationality ,  COUNT(*) FROM people GROUP BY Nationality", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Nationality FROM people GROUP BY people.Nationality ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Nationality FROM people GROUP BY Nationality ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT people.Nationality FROM people GROUP BY people.Nationality", "gold": "SELECT Nationality FROM people GROUP BY Nationality ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT people.Nationality FROM people GROUP BY people.Nationality HAVING Count(*) >= 'terminal'", "gold": "SELECT Nationality FROM people GROUP BY Nationality HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Nationality FROM people GROUP BY people.Nationality HAVING Count(*) >= 'terminal'", "gold": "SELECT Nationality FROM people GROUP BY Nationality HAVING COUNT(*)  >=  2", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Name, people.Birth_Date FROM people ORDER BY people.Name Asc", "gold": "SELECT Name ,  Birth_Date FROM people ORDER BY Name ASC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Name, people.Birth_Date FROM people ORDER BY people.Name Asc", "gold": "SELECT Name ,  Birth_Date FROM people ORDER BY Name ASC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT people.Name FROM people WHERE people.Nationality != 'terminal'", "gold": "SELECT Name FROM people WHERE Nationality != \"Russia\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM people WHERE people.Nationality != 'terminal'", "gold": "SELECT Name FROM people WHERE Nationality != \"Russia\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM people EXCEPT SELECT people.Name FROM poker_player JOIN people WHERE poker_player.Poker_Player_ID = 'terminal'", "gold": "SELECT Name FROM people WHERE People_ID NOT IN (SELECT People_ID FROM poker_player)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT people.Name FROM people WHERE people.People_ID NOT IN (SELECT poker_player.People_ID FROM poker_player)", "gold": "SELECT Name FROM people WHERE People_ID NOT IN (SELECT People_ID FROM poker_player)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(DISTINCT people.Nationality) FROM people", "gold": "SELECT count(DISTINCT Nationality) FROM people", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT people.Nationality) FROM people", "gold": "SELECT count(DISTINCT Nationality) FROM people", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM AREA_CODE_STATE", "gold": "SELECT count(*) FROM area_code_state", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT CONTESTANTS.contestant_number, CONTESTANTS.contestant_name FROM CONTESTANTS ORDER BY CONTESTANTS.contestant_name Desc", "gold": "SELECT contestant_number ,  contestant_name FROM contestants ORDER BY contestant_name DESC", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT VOTES.vote_id, VOTES.phone_number, VOTES.state FROM VOTES", "gold": "SELECT vote_id ,  phone_number ,  state FROM votes", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Max(AREA_CODE_STATE.area_code), Min(AREA_CODE_STATE.area_code) FROM AREA_CODE_STATE", "gold": "SELECT max(area_code) ,  min(area_code) FROM area_code_state", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT VOTES.vote_id FROM VOTES WHERE VOTES.state = 'terminal'", "gold": "SELECT max(created) FROM votes WHERE state  =  'CA'", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT VOTES.vote_id FROM CONTESTANTS JOIN VOTES WHERE CONTESTANTS.contestant_name != 'terminal'", "gold": "SELECT contestant_name FROM contestants WHERE contestant_name != 'Jessie Alloway'", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT VOTES.state, VOTES.created FROM VOTES", "gold": "SELECT DISTINCT state ,  created FROM votes", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT CONTESTANTS.contestant_number, CONTESTANTS.contestant_name FROM VOTES JOIN CONTESTANTS GROUP BY VOTES.contestant_number HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.contestant_number , T1.contestant_name FROM contestants AS T1 JOIN votes AS T2 ON T1.contestant_number  =  T2.contestant_number GROUP BY T1.contestant_number HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*), CONTESTANTS.contestant_name FROM CONTESTANTS JOIN VOTES ORDER BY VOTES.created Asc LIMIT 1", "gold": "SELECT T1.contestant_number , T1.contestant_name FROM contestants AS T1 JOIN votes AS T2 ON T1.contestant_number  =  T2.contestant_number GROUP BY T1.contestant_number ORDER BY count(*) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM VOTES WHERE VOTES.state = 'terminal' OR VOTES.state = 'terminal'", "gold": "SELECT count(*) FROM votes WHERE state  =  'NY' OR state  =  'CA'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM CONTESTANTS WHERE CONTESTANTS.contestant_number NOT IN (SELECT VOTES.vote_id FROM VOTES)", "gold": "SELECT count(*) FROM contestants WHERE contestant_number NOT IN ( SELECT contestant_number FROM votes )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT AREA_CODE_STATE.area_code FROM VOTES JOIN AREA_CODE_STATE GROUP BY AREA_CODE_STATE.area_code ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.area_code FROM area_code_state AS T1 JOIN votes AS T2 ON T1.state  =  T2.state GROUP BY T1.area_code ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT VOTES.state, VOTES.state, VOTES.phone_number FROM CONTESTANTS JOIN VOTES WHERE CONTESTANTS.contestant_name = 'terminal'", "gold": "SELECT T2.created ,  T2.state ,  T2.phone_number FROM contestants AS T1 JOIN votes AS T2 ON T1.contestant_number  =  T2.contestant_number WHERE T1.contestant_name  =  'Tabatha Gehling'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT AREA_CODE_STATE.area_code FROM CONTESTANTS JOIN VOTES JOIN AREA_CODE_STATE WHERE CONTESTANTS.contestant_name = 'terminal' INTERSECT SELECT AREA_CODE_STATE.area_code FROM CONTESTANTS JOIN VOTES JOIN AREA_CODE_STATE WHERE CONTESTANTS.contestant_name = 'terminal'", "gold": "SELECT T3.area_code FROM contestants AS T1 JOIN votes AS T2 ON T1.contestant_number  =  T2.contestant_number JOIN area_code_state AS T3 ON T2.state  =  T3.state WHERE T1.contestant_name  =  'Tabatha Gehling' INTERSECT SELECT T3.area_code FROM contestants AS T1 JOIN votes AS T2 ON T1.contestant_number  =  T2.contestant_number JOIN area_code_state AS T3 ON T2.state  =  T3.state WHERE T1.contestant_name  =  'Kelly Clauss'", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT CONTESTANTS.contestant_name FROM CONTESTANTS WHERE CONTESTANTS.contestant_name LIKE 'terminal'", "gold": "select contestant_name from contestants where contestant_name like \"%al%\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name FROM country WHERE country.IndepYear > 'terminal'", "gold": "SELECT Name FROM country WHERE IndepYear  >  1950", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.IndepYear > 'terminal'", "gold": "SELECT Name FROM country WHERE IndepYear  >  1950", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM country", "gold": "SELECT count(*) FROM country WHERE GovernmentForm  =  \"Republic\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT count(*) FROM country WHERE GovernmentForm  =  \"Republic\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.SurfaceArea) FROM country WHERE country.Name = 'terminal'", "gold": "SELECT sum(SurfaceArea) FROM country WHERE Region  =  \"Caribbean\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.SurfaceArea) FROM country WHERE country.Name = 'terminal'", "gold": "SELECT sum(SurfaceArea) FROM country WHERE Region  =  \"Caribbean\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.District FROM city WHERE city.Name = 'terminal'", "gold": "SELECT Continent FROM country WHERE Name  =  \"Anguilla\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM city JOIN country WHERE city.Name = 'terminal'", "gold": "SELECT Continent FROM country WHERE Name  =  \"Anguilla\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.District FROM city WHERE city.Name = 'terminal'", "gold": "SELECT Region FROM country AS T1 JOIN city AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Name  =  \"Kabul\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Region FROM country WHERE country.Region = 'terminal'", "gold": "SELECT Region FROM country AS T1 JOIN city AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Name  =  \"Kabul\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.Name FROM city JOIN country WHERE country.Name = 'terminal' GROUP BY city.Name ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Aruba\" ORDER BY Percentage DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 5}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage JOIN country WHERE country.Name = 'terminal' GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Aruba\" ORDER BY Percentage DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 5}}}, {"predicted": "SELECT Sum(city.Population), city.Population FROM city", "gold": "SELECT Population ,  LifeExpectancy FROM country WHERE Name  =  \"Brazil\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT country.Population, country.HeadOfState FROM country", "gold": "SELECT Population ,  LifeExpectancy FROM country WHERE Name  =  \"Brazil\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT country.Region, country.Population FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT Population ,  Region FROM country WHERE Name  =  \"Angola\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Region, country.Population FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT Population ,  Region FROM country WHERE Name  =  \"Angola\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(country.HeadOfState) FROM country WHERE country.Region = 'terminal'", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Region  =  \"Central Africa\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(city.Population) FROM city WHERE city.Name = 'terminal'", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Region  =  \"Central Africa\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country JOIN city ORDER BY city.Population Asc LIMIT 1", "gold": "SELECT Name FROM country WHERE Continent  =  \"Asia\" ORDER BY LifeExpectancy LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT country.Name FROM country JOIN city WHERE city.CountryCode = 'terminal' ORDER BY country.HeadOfState Asc LIMIT 1", "gold": "SELECT Name FROM country WHERE Continent  =  \"Asia\" ORDER BY LifeExpectancy LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Sum(Sum(country.Population)), Max(city.Population) FROM country JOIN city WHERE city.CountryCode = 'terminal'", "gold": "SELECT sum(Population) ,  max(GNP) FROM country WHERE Continent  =  \"Asia\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.Population), Max(country.Population) FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT sum(Population) ,  max(GNP) FROM country WHERE Continent  =  \"Asia\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(country.Population) FROM country WHERE country.Capital = 'terminal'", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Continent  =  \"Africa\" AND GovernmentForm  =  \"Republic\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(country.HeadOfState) FROM country WHERE country.Capital = 'terminal'", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Continent  =  \"Africa\" AND GovernmentForm  =  \"Republic\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.SurfaceArea) FROM country WHERE country.Region = 'terminal'", "gold": "SELECT sum(SurfaceArea) FROM country WHERE Continent  =  \"Asia\" OR Continent  =  \"Europe\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.SurfaceArea) FROM country WHERE country.Capital = 'terminal' OR country.Capital = 'terminal'", "gold": "SELECT sum(SurfaceArea) FROM country WHERE Continent  =  \"Asia\" OR Continent  =  \"Europe\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT city.Population FROM city WHERE city.District = 'terminal'", "gold": "SELECT sum(Population) FROM city WHERE District  =  \"Gelderland\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(city.Population) FROM city WHERE city.District = 'terminal'", "gold": "SELECT sum(Population) FROM city WHERE District  =  \"Gelderland\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(country.GNP), Sum(country.Population) FROM country WHERE country.HeadOfState = 'terminal'", "gold": "SELECT avg(GNP) ,  sum(population) FROM country WHERE GovernmentForm  =  \"US Territory\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(country.GNP), Sum(country.Population) FROM country JOIN city WHERE city.Name = 'terminal'", "gold": "SELECT avg(GNP) ,  sum(population) FROM country WHERE GovernmentForm  =  \"US Territory\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage", "gold": "SELECT count(DISTINCT LANGUAGE) FROM countrylanguage", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage", "gold": "SELECT count(DISTINCT LANGUAGE) FROM countrylanguage", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT country.GovernmentForm) FROM country WHERE country.GovernmentForm = 'terminal'", "gold": "SELECT count(DISTINCT GovernmentForm) FROM country WHERE Continent  =  \"Africa\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT country.GovernmentForm) FROM country WHERE country.GovernmentForm = 'terminal'", "gold": "SELECT count(DISTINCT GovernmentForm) FROM country WHERE Continent  =  \"Africa\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countrylanguage JOIN country WHERE country.LocalName = 'terminal'", "gold": "SELECT COUNT(T2.Language) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Aruba\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM city JOIN country WHERE country.Name = 'terminal' AND country.Name = 'terminal'", "gold": "SELECT COUNT(T2.Language) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Aruba\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT(*) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Afghanistan\" AND IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT(*) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Name  =  \"Afghanistan\" AND IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country JOIN countrylanguage GROUP BY country.Name ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Name ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT country.Name FROM countrylanguage JOIN country GROUP BY country.Name ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Name ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Continent FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Continent ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Continent FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Continent ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Count(*) FROM country JOIN countrylanguage WHERE countrylanguage.Language = 'terminal' AND countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT(*) FROM (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"Dutch\")", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT(*) FROM (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"Dutch\")", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.LocalName = 'terminal' INTERSECT SELECT country.Name FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"French\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name FROM country WHERE country.LocalName = 'terminal' INTERSECT SELECT country.Name FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"French\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name FROM countrylanguage JOIN country WHERE countrylanguage.Language = 'terminal' INTERSECT SELECT country.Name FROM countrylanguage JOIN country WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" AND T2.IsOfficial  =  \"T\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"French\" AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name FROM countrylanguage JOIN country WHERE countrylanguage.Language = 'terminal' INTERSECT SELECT country.Name FROM countrylanguage JOIN country WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" AND T2.IsOfficial  =  \"T\" INTERSECT SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"French\" AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT( DISTINCT Continent) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"Chinese\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT COUNT( DISTINCT Continent) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"Chinese\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage WHERE countrylanguage.Language = 'terminal' OR countrylanguage.Language = 'terminal'", "gold": "SELECT DISTINCT T1.Region FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" OR T2.Language  =  \"Dutch\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT city.Name FROM countrylanguage JOIN city WHERE countrylanguage.Language = 'terminal' OR countrylanguage.Language = 'terminal'", "gold": "SELECT DISTINCT T1.Region FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" OR T2.Language  =  \"Dutch\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage WHERE countrylanguage.Language = 'terminal' OR countrylanguage.Language = 'terminal'", "gold": "select t1.name from country as t1 join countrylanguage as t2 on t1.code  =  t2.countrycode where t2.language  =  \"english\" and isofficial  =  \"t\" union select t1.name from country as t1 join countrylanguage as t2 on t1.code  =  t2.countrycode where t2.language  =  \"dutch\" and isofficial  =  \"t\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name FROM country JOIN countrylanguage WHERE countrylanguage.Language = 'terminal' OR countrylanguage.Language = 'terminal'", "gold": "SELECT * FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" AND IsOfficial  =  \"T\" UNION SELECT * FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"Dutch\" AND IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Continent  =  \"Asia\" GROUP BY T2.Language ORDER BY COUNT (*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 5, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.Continent  =  \"Asia\" GROUP BY T2.Language ORDER BY COUNT (*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 5, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language HAVING Count(*) = 'terminal'", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.GovernmentForm  =  \"Republic\" GROUP BY T2.Language HAVING COUNT(*)  =  1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language HAVING Count(*) = 'terminal'", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.GovernmentForm  =  \"Republic\" GROUP BY T2.Language HAVING COUNT(*)  =  1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT city.Name FROM city JOIN countrylanguage WHERE countrylanguage.Language = 'terminal' ORDER BY city.Population Desc LIMIT 1", "gold": "SELECT T1.Name ,  T1.Population FROM city AS T1 JOIN countrylanguage AS T2 ON T1.CountryCode  =  T2.CountryCode WHERE T2.Language  =  \"English\" ORDER BY T1.Population DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT city.Name FROM city JOIN countrylanguage WHERE countrylanguage.Language = 'terminal' ORDER BY city.Population Desc LIMIT 1", "gold": "SELECT T1.Name ,  T1.Population FROM city AS T1 JOIN countrylanguage AS T2 ON T1.CountryCode  =  T2.CountryCode WHERE T2.Language  =  \"English\" ORDER BY T1.Population DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT country.Name, country.Population, city.Population FROM country JOIN city WHERE city.CountryCode = 'terminal' ORDER BY country.SurfaceArea Desc LIMIT 1", "gold": "SELECT Name ,  Population ,  LifeExpectancy FROM country WHERE Continent  =  \"Asia\" ORDER BY SurfaceArea DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT country.Name, country.Population, city.Population FROM country JOIN city ORDER BY country.SurfaceArea Desc LIMIT 1", "gold": "SELECT Name ,  Population ,  LifeExpectancy FROM country WHERE Continent  =  \"Asia\" ORDER BY SurfaceArea DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT Avg(country.HeadOfState) FROM country WHERE country.Code NOT IN (SELECT countrylanguage.CountryCode FROM countrylanguage WHERE countrylanguage.Language = 'terminal')", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Name NOT IN (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" AND T2.IsOfficial  =  \"T\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Avg(country.HeadOfState) FROM country WHERE country.Code NOT IN (SELECT countrylanguage.Language FROM countrylanguage)", "gold": "SELECT avg(LifeExpectancy) FROM country WHERE Name NOT IN (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\" AND T2.IsOfficial  =  \"T\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Sum(country.Population) FROM country WHERE country.Code NOT IN (SELECT countrylanguage.Language FROM countrylanguage WHERE countrylanguage.Language = 'terminal')", "gold": "SELECT sum(Population) FROM country WHERE Name NOT IN (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Sum(country.Population) FROM country WHERE city.CountryCode NOT IN (SELECT countrylanguage.Language FROM countrylanguage WHERE countrylanguage.Language = 'terminal')", "gold": "SELECT sum(Population) FROM country WHERE Name NOT IN (SELECT T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T2.Language  =  \"English\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT countrylanguage.Language FROM country JOIN countrylanguage WHERE country.HeadOfState = 'terminal'", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.HeadOfState  =  \"Beatrix\" AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countrylanguage.Language FROM country JOIN countrylanguage WHERE country.HeadOfState = 'terminal'", "gold": "SELECT T2.Language FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE T1.HeadOfState  =  \"Beatrix\" AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage JOIN country WHERE country.IndepYear < 'terminal'", "gold": "SELECT count(DISTINCT T2.Language) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE  IndepYear  <  1930 AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(DISTINCT countrylanguage.Language) FROM countrylanguage JOIN country WHERE country.IndepYear < 'terminal'", "gold": "SELECT count(DISTINCT T2.Language) FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode WHERE  IndepYear  <  1930 AND T2.IsOfficial  =  \"T\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.SurfaceArea > (SELECT Max(country.SurfaceArea) FROM country)", "gold": "SELECT Name FROM country WHERE SurfaceArea  >  (SELECT min(SurfaceArea) FROM country WHERE Continent  =  \"Europe\")", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.SurfaceArea > (SELECT country.SurfaceArea FROM country)", "gold": "SELECT Name FROM country WHERE SurfaceArea  >  (SELECT min(SurfaceArea) FROM country WHERE Continent  =  \"Europe\")", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.Population < (SELECT Min(country.Population) FROM city)", "gold": "SELECT Name FROM country WHERE Continent  =  \"Africa\"  AND population  <  (SELECT max(population) FROM country WHERE Continent  =  \"Asia\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.Population < (SELECT Min(country.Population) FROM country)", "gold": "SELECT Name FROM country WHERE Continent  =  \"Africa\"  AND population  <  (SELECT min(population) FROM country WHERE Continent  =  \"Asia\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE city.Population > (SELECT Max(city.Population) FROM city)", "gold": "SELECT Name FROM country WHERE Continent  =  \"Asia\"  AND population  >  (SELECT max(population) FROM country WHERE Continent  =  \"Africa\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.Name FROM city WHERE city.Population > (SELECT Max(city.Population) FROM city)", "gold": "SELECT Name FROM country WHERE Continent  =  \"Asia\"  AND population  >  (SELECT min(population) FROM country WHERE Continent  =  \"Africa\")", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country EXCEPT SELECT countrylanguage.CountryCode FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT CountryCode FROM countrylanguage EXCEPT SELECT CountryCode FROM countrylanguage WHERE LANGUAGE  =  \"English\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country EXCEPT SELECT countrylanguage.CountryCode FROM countrylanguage", "gold": "SELECT CountryCode FROM countrylanguage EXCEPT SELECT CountryCode FROM countrylanguage WHERE LANGUAGE  =  \"English\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country JOIN countrylanguage WHERE countrylanguage.Language > 'terminal'", "gold": "SELECT DISTINCT CountryCode FROM countrylanguage WHERE LANGUAGE != \"English\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country JOIN countrylanguage WHERE countrylanguage.Language != 'terminal'", "gold": "SELECT DISTINCT CountryCode FROM countrylanguage WHERE LANGUAGE != \"English\"", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country WHERE country.Code NOT IN (SELECT countrylanguage.Language FROM countrylanguage)", "gold": "SELECT Code FROM country WHERE GovernmentForm != \"Republic\" EXCEPT SELECT CountryCode FROM countrylanguage WHERE LANGUAGE  =  \"English\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT country.Code FROM country EXCEPT SELECT countrylanguage.CountryCode FROM countrylanguage", "gold": "SELECT Code FROM country WHERE GovernmentForm != \"Republic\" EXCEPT SELECT CountryCode FROM countrylanguage WHERE LANGUAGE  =  \"English\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT city.Name FROM city WHERE city.ID NOT IN (SELECT countrylanguage.CountryCode FROM countrylanguage WHERE countrylanguage.Language = 'terminal')", "gold": "SELECT DISTINCT T2.Name FROM country AS T1 JOIN city AS T2 ON T2.CountryCode  =  T1.Code WHERE T1.Continent  =  'Europe' AND T1.Name NOT IN (SELECT T3.Name FROM country AS T3 JOIN countrylanguage AS T4 ON T3.Code  =  T4.CountryCode WHERE T4.IsOfficial  =  'T' AND T4.Language  =  'English')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT city.Name FROM city JOIN countrylanguage WHERE countrylanguage.Language != 'terminal'", "gold": "SELECT DISTINCT T2.Name FROM country AS T1 JOIN city AS T2 ON T2.CountryCode  =  T1.Code WHERE T1.Continent  =  'Europe' AND T1.Name NOT IN (SELECT T3.Name FROM country AS T3 JOIN countrylanguage AS T4 ON T3.Code  =  T4.CountryCode WHERE T4.IsOfficial  =  'T' AND T4.Language  =  'English')", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT city.Name FROM city JOIN countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "select distinct t3.name from country as t1 join countrylanguage as t2 on t1.code  =  t2.countrycode join city as t3 on t1.code  =  t3.countrycode where t2.isofficial  =  't' and t2.language  =  'chinese' and t1.continent  =  \"asia\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT city.Name FROM city WHERE city.CountryCode = 'terminal'", "gold": "SELECT DISTINCT T3.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode JOIN city AS T3 ON T1.Code  =  T3.CountryCode WHERE T2.IsOfficial  =  'T' AND T2.Language  =  'Chinese' AND T1.Continent  =  \"Asia\"", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name, country.Continent, country.SurfaceArea FROM country ORDER BY country.Population Asc LIMIT 1", "gold": "SELECT Name ,  SurfaceArea ,  IndepYear FROM country ORDER BY Population LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name, country.Continent, country.Region FROM country JOIN city ORDER BY city.Population Asc LIMIT 1", "gold": "SELECT Name ,  SurfaceArea ,  IndepYear FROM country ORDER BY Population LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Population, country.Name, country.SurfaceArea FROM country ORDER BY country.SurfaceArea Desc LIMIT 1", "gold": "SELECT Name ,  population ,  HeadOfState FROM country ORDER BY SurfaceArea DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name, country.Population, country.HeadOfState FROM country ORDER BY country.SurfaceArea Desc LIMIT 1", "gold": "SELECT Name ,  population ,  HeadOfState FROM country ORDER BY SurfaceArea DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name, Count(*) FROM countrylanguage JOIN country GROUP BY countrylanguage.Language HAVING Count(*) >= 'terminal'", "gold": "SELECT COUNT(T2.Language) ,  T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Name HAVING COUNT(*)  >  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT country.Name, Count(*) FROM country JOIN countrylanguage GROUP BY country.Name HAVING Count(*) > 'terminal'", "gold": "SELECT COUNT(T2.Language) ,  T1.Name FROM country AS T1 JOIN countrylanguage AS T2 ON T1.Code  =  T2.CountryCode GROUP BY T1.Name HAVING COUNT(*)  >  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*), city.District FROM city WHERE city.Population > (SELECT Avg(city.Population) FROM city)", "gold": "SELECT count(*) ,  District FROM city WHERE Population  >  (SELECT avg(Population) FROM city) GROUP BY District", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Count(*), city.District FROM city WHERE city.Population > (SELECT Avg(city.Population) FROM city)", "gold": "SELECT count(*) ,  District FROM city WHERE Population  >  (SELECT avg(Population) FROM city) GROUP BY District", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT country.Name, Sum(city.Population) FROM city WHERE city.Population > (SELECT Avg(city.Population) FROM city)", "gold": "SELECT sum(Population) ,  GovernmentForm FROM country GROUP BY GovernmentForm HAVING avg(LifeExpectancy)  >  72", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.Population), country.HeadOfState FROM country JOIN city GROUP BY country.HeadOfState HAVING Avg(city.Population) > 'terminal'", "gold": "SELECT sum(Population) ,  GovernmentForm FROM country GROUP BY GovernmentForm HAVING avg(LifeExpectancy)  >  72", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Avg(city.Population), Sum(city.Population) FROM city", "gold": "SELECT sum(Population) ,  avg(LifeExpectancy) ,  Continent FROM country GROUP BY Continent HAVING avg(LifeExpectancy)  <  72", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Sum(country.Population), Avg(city.Population) FROM country JOIN city", "gold": "SELECT sum(Population) ,  avg(LifeExpectancy) ,  Continent FROM country GROUP BY Continent HAVING avg(LifeExpectancy)  <  72", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT country.Name, country.Region FROM country ORDER BY country.SurfaceArea Desc LIMIT 1", "gold": "SELECT Name ,  SurfaceArea FROM country ORDER BY SurfaceArea DESC LIMIT 5", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name, country.SurfaceArea FROM country ORDER BY country.Continent Desc LIMIT 1", "gold": "SELECT Name ,  SurfaceArea FROM country ORDER BY SurfaceArea DESC LIMIT 5", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name FROM country JOIN city ORDER BY country.Population Desc LIMIT 1", "gold": "SELECT Name FROM country ORDER BY Population DESC LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name FROM country ORDER BY country.Population Desc LIMIT 1", "gold": "SELECT Name FROM country ORDER BY Population DESC LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name FROM country JOIN city ORDER BY city.Population Asc LIMIT 1", "gold": "SELECT Name FROM country ORDER BY Population ASC LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT country.Name FROM city JOIN country ORDER BY city.Population Asc LIMIT 1", "gold": "SELECT Name FROM country ORDER BY Population ASC LIMIT 3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM country", "gold": "SELECT count(*) FROM country WHERE continent  =  \"Asia\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM country WHERE country.Capital = 'terminal'", "gold": "SELECT count(*) FROM country WHERE continent  =  \"Asia\"", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country WHERE country.LocalName = 'terminal'", "gold": "SELECT Name FROM country WHERE continent  =  \"Europe\" AND Population  =  \"80000\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT country.Name FROM country JOIN city WHERE country.LocalName = 'terminal'", "gold": "SELECT Name FROM country WHERE continent  =  \"Europe\" AND Population  =  \"80000\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.Population), Avg(country.SurfaceArea) FROM country JOIN city WHERE country.SurfaceArea > 'terminal'", "gold": "select sum(population) ,  avg(surfacearea) from country where continent  =  \"north america\" and surfacearea  >  3000", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(country.Population), Avg(country.SurfaceArea) FROM country WHERE country.SurfaceArea > 'terminal'", "gold": "select sum(population) ,  avg(surfacearea) from country where continent  =  \"north america\" and surfacearea  >  3000", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.Name FROM city WHERE city.Population BETWEEN 'terminal' AND 'terminal'", "gold": "SELECT name FROM city WHERE Population BETWEEN 160000 AND 900000", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT city.Name FROM city WHERE city.Population BETWEEN 'terminal' AND 'terminal'", "gold": "select name from city where population between 160000 and 900000", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT LANGUAGE FROM countrylanguage GROUP BY LANGUAGE ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT LANGUAGE FROM countrylanguage GROUP BY LANGUAGE ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT countrylanguage.Language, countrylanguage.Language FROM countrylanguage JOIN country GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT LANGUAGE ,  CountryCode ,  max(Percentage) FROM countrylanguage GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 4}}}, {"predicted": "SELECT city.CountryCode, countrylanguage.Language FROM countrylanguage JOIN city GROUP BY city.CountryCode ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT LANGUAGE ,  CountryCode ,  max(Percentage) FROM countrylanguage GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 4}}}, {"predicted": "SELECT Count(*) FROM countrylanguage WHERE countrylanguage.Language = (SELECT countrylanguage.Language FROM countrylanguage GROUP BY countrylanguage.Language ORDER BY Count(*) Desc LIMIT 1)", "gold": "SELECT count(*) ,   max(Percentage) FROM countrylanguage WHERE LANGUAGE  =  \"Spanish\" GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM countrylanguage WHERE countrylanguage.Language = 'terminal'", "gold": "SELECT count(*) ,   max(Percentage) FROM countrylanguage WHERE LANGUAGE  =  \"Spanish\" GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}}}, {"predicted": "SELECT country.Code FROM country WHERE countrylanguage.Language = (SELECT city.CountryCode FROM country GROUP BY city.CountryCode ORDER BY Count(*) Desc LIMIT 1) ORDER BY country.Population Desc LIMIT 1", "gold": "SELECT CountryCode ,  max(Percentage) FROM countrylanguage WHERE LANGUAGE  =  \"Spanish\" GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 4}}}, {"predicted": "SELECT country.Code FROM country JOIN countrylanguage WHERE countrylanguage.Language = 'terminal' GROUP BY countrylanguage.CountryCode ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT CountryCode ,  max(Percentage) FROM countrylanguage WHERE LANGUAGE  =  \"Spanish\" GROUP BY CountryCode", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 5}}}, {"predicted": "SELECT Count(*) FROM conductor", "gold": "SELECT count(*) FROM conductor", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM orchestra", "gold": "SELECT count(*) FROM conductor", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT conductor.Name FROM conductor ORDER BY conductor.Age Asc", "gold": "SELECT Name FROM conductor ORDER BY Age ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor ORDER BY conductor.Age Asc", "gold": "SELECT Name FROM conductor ORDER BY Age ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor WHERE conductor.Nationality != 'terminal'", "gold": "SELECT Name FROM conductor WHERE Nationality != 'USA'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT conductor.Name FROM conductor WHERE conductor.Nationality != 'terminal'", "gold": "SELECT Name FROM conductor WHERE Nationality != 'USA'", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra ORDER BY orchestra.Year_of_Founded Desc", "gold": "SELECT Record_Company FROM orchestra ORDER BY Year_of_Founded DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT orchestra.Record_Company FROM orchestra ORDER BY orchestra.Year_of_Founded Asc", "gold": "SELECT Record_Company FROM orchestra ORDER BY Year_of_Founded DESC", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Avg(show.Attendance) FROM show", "gold": "SELECT avg(Attendance) FROM SHOW", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(performance.Share) FROM show JOIN performance", "gold": "SELECT avg(Attendance) FROM SHOW", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Max(performance.Official_ratings_(millions)), Min(performance.Official_ratings_(millions)) FROM performance WHERE performance.Type != 'terminal'", "gold": "SELECT max(SHARE) ,  min(SHARE) FROM performance WHERE TYPE != \"Live final\"", "predicted_parse_error": true, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Max(performance.Official_ratings_(millions)), Min(performance.Official_ratings_(millions)) FROM performance WHERE performance.Type != 'terminal'", "gold": "SELECT max(SHARE) ,  min(SHARE) FROM performance WHERE TYPE != \"Live final\"", "predicted_parse_error": true, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT conductor.Nationality) FROM conductor", "gold": "SELECT count(DISTINCT Nationality) FROM conductor", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT conductor.Nationality) FROM conductor", "gold": "SELECT count(DISTINCT Nationality) FROM conductor", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT conductor.Name FROM conductor ORDER BY conductor.Year_of_Work Desc", "gold": "SELECT Name FROM conductor ORDER BY Year_of_Work DESC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor ORDER BY conductor.Year_of_Work Desc", "gold": "SELECT Name FROM conductor ORDER BY Year_of_Work DESC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra ORDER BY conductor.Year_of_Work Desc LIMIT 1", "gold": "SELECT Name FROM conductor ORDER BY Year_of_Work DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra ORDER BY conductor.Year_of_Work Desc LIMIT 1", "gold": "SELECT Name FROM conductor ORDER BY Year_of_Work DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT conductor.Name, conductor.Name FROM conductor JOIN orchestra", "gold": "SELECT T1.Name ,  T2.Orchestra FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT conductor.Name, orchestra.Orchestra FROM conductor JOIN orchestra", "gold": "SELECT T1.Name ,  T2.Orchestra FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra GROUP BY orchestra.Conductor_ID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T2.Conductor_ID HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra GROUP BY orchestra.Conductor_ID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T2.Conductor_ID HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra GROUP BY orchestra.Conductor_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T2.Conductor_ID ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra GROUP BY orchestra.Conductor_ID ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID GROUP BY T2.Conductor_ID ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra WHERE orchestra.Year_of_Founded > 'terminal'", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID WHERE Year_of_Founded  >  2008", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT conductor.Name FROM conductor JOIN orchestra WHERE orchestra.Year_of_Founded > 'terminal'", "gold": "SELECT T1.Name FROM conductor AS T1 JOIN orchestra AS T2 ON T1.Conductor_ID  =  T2.Conductor_ID WHERE Year_of_Founded  >  2008", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT orchestra.Record_Company, Count(*) FROM orchestra GROUP BY orchestra.Record_Company", "gold": "SELECT Record_Company ,  COUNT(*) FROM orchestra GROUP BY Record_Company", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT orchestra.Orchestra, Count(*) FROM orchestra GROUP BY orchestra.Orchestra", "gold": "SELECT Record_Company ,  COUNT(*) FROM orchestra GROUP BY Record_Company", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra ORDER BY orchestra.Year_of_Founded Asc", "gold": "SELECT Major_Record_Format FROM orchestra GROUP BY Major_Record_Format ORDER BY COUNT(*) ASC", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT performance.Type FROM performance ORDER BY performance.Type Asc", "gold": "SELECT Major_Record_Format FROM orchestra GROUP BY Major_Record_Format ORDER BY COUNT(*) ASC", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 2}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra GROUP BY orchestra.Orchestra ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Record_Company FROM orchestra GROUP BY Record_Company ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra GROUP BY orchestra.Orchestra ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Record_Company FROM orchestra GROUP BY Record_Company ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra WHERE orchestra.Orchestra_ID NOT IN (SELECT performance.Orchestra_ID FROM performance)", "gold": "SELECT Orchestra FROM orchestra WHERE Orchestra_ID NOT IN (SELECT Orchestra_ID FROM performance)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT orchestra.Orchestra FROM orchestra EXCEPT SELECT orchestra.Orchestra FROM orchestra JOIN performance", "gold": "SELECT Orchestra FROM orchestra WHERE Orchestra_ID NOT IN (SELECT Orchestra_ID FROM performance)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT orchestra.Record_Company FROM orchestra WHERE orchestra.Year_of_Founded < 'terminal' INTERSECT SELECT orchestra.Record_Company FROM orchestra WHERE orchestra.Year_of_Founded > 'terminal'", "gold": "SELECT Record_Company FROM orchestra WHERE Year_of_Founded  <  2003 INTERSECT SELECT Record_Company FROM orchestra WHERE Year_of_Founded  >  2003", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT orchestra.Record_Company FROM orchestra WHERE orchestra.Year_of_Founded < 'terminal' INTERSECT SELECT orchestra.Record_Company FROM orchestra WHERE orchestra.Year_of_Founded > 'terminal'", "gold": "SELECT Record_Company FROM orchestra WHERE Year_of_Founded  <  2003 INTERSECT SELECT Record_Company FROM orchestra WHERE Year_of_Founded  >  2003", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM orchestra WHERE orchestra.Major_Record_Format = 'terminal' OR orchestra.Major_Record_Format = 'terminal'", "gold": "SELECT COUNT(*) FROM orchestra WHERE Major_Record_Format  =  \"CD\" OR Major_Record_Format  =  \"DVD\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM orchestra WHERE orchestra.Major_Record_Format = 'terminal' OR orchestra.Major_Record_Format = 'terminal'", "gold": "SELECT COUNT(*) FROM orchestra WHERE Major_Record_Format  =  \"CD\" OR Major_Record_Format  =  \"DVD\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT orchestra.Year_of_Founded FROM performance JOIN orchestra GROUP BY orchestra.Year_of_Founded HAVING Count(*) > 'terminal'", "gold": "SELECT Year_of_Founded FROM orchestra AS T1 JOIN performance AS T2 ON T1.Orchestra_ID  =  T2.Orchestra_ID GROUP BY T2.Orchestra_ID HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT performance.Weekly_rank FROM performance GROUP BY performance.Orchestra_ID HAVING Count(*) > 'terminal'", "gold": "SELECT Year_of_Founded FROM orchestra AS T1 JOIN performance AS T2 ON T1.Orchestra_ID  =  T2.Orchestra_ID GROUP BY T2.Orchestra_ID HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT count(*) FROM Highschooler", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT count(*) FROM Highschooler", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name, Highschooler.grade FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT name ,  grade FROM Highschooler", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name, Highschooler.grade FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT name ,  grade FROM Highschooler", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler", "gold": "SELECT grade FROM Highschooler", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler", "gold": "SELECT grade FROM Highschooler", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler WHERE Highschooler.name = 'terminal'", "gold": "SELECT grade FROM Highschooler WHERE name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler WHERE Highschooler.name = 'terminal'", "gold": "SELECT grade FROM Highschooler WHERE name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT name FROM Highschooler WHERE grade  =  10", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT name FROM Highschooler WHERE grade  =  10", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.ID FROM Highschooler JOIN Friend WHERE Highschooler.name = 'terminal'", "gold": "SELECT ID FROM Highschooler WHERE name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.ID FROM Highschooler WHERE Highschooler.name = 'terminal'", "gold": "SELECT ID FROM Highschooler WHERE name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Highschooler WHERE Highschooler.grade = 'terminal' OR Highschooler.grade = 'terminal'", "gold": "SELECT count(*) FROM Highschooler WHERE grade  =  9 OR grade  =  10", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM Highschooler WHERE Highschooler.grade = 'terminal' OR Highschooler.grade = 'terminal'", "gold": "SELECT count(*) FROM Highschooler WHERE grade  =  9 OR grade  =  10", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*), Highschooler.grade FROM Highschooler GROUP BY Highschooler.grade", "gold": "SELECT grade ,  count(*) FROM Highschooler GROUP BY grade", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Highschooler.grade FROM Highschooler GROUP BY Highschooler.grade", "gold": "SELECT grade ,  count(*) FROM Highschooler GROUP BY grade", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler GROUP BY Highschooler.grade ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT grade FROM Highschooler GROUP BY grade ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler GROUP BY Highschooler.grade ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT grade FROM Highschooler GROUP BY grade ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler JOIN Friend GROUP BY Highschooler.grade HAVING Count(*) >= 'terminal'", "gold": "SELECT grade FROM Highschooler GROUP BY grade HAVING count(*)  >=  4", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Highschooler.grade FROM Friend JOIN Highschooler GROUP BY Highschooler.grade HAVING Count(*) >= 'terminal'", "gold": "SELECT grade FROM Highschooler GROUP BY grade HAVING count(*)  >=  4", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Friend.student_id, Count(*) FROM Friend GROUP BY Friend.student_id", "gold": "SELECT student_id ,  count(*) FROM Friend GROUP BY student_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Friend.student_id FROM Highschooler JOIN Friend GROUP BY Friend.student_id", "gold": "SELECT student_id ,  count(*) FROM Friend GROUP BY student_id", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name, Count(*) FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal'", "gold": "SELECT T2.name ,  count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*), Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal' GROUP BY Friend.student_id", "gold": "SELECT T2.name ,  count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend GROUP BY Friend.student_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = (SELECT Friend.friend_id FROM Highschooler JOIN Friend GROUP BY Friend.student_id ORDER BY Count(*) Desc LIMIT 1)", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal' GROUP BY Highschooler.name HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id HAVING count(*)  >=  3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal' GROUP BY Friend.student_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id HAVING count(*)  >=  3", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT DISTINCT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.name = 'terminal'", "gold": "SELECT T3.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id JOIN Highschooler AS T3 ON T1.friend_id  =  T3.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.name = 'terminal'", "gold": "SELECT T3.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id JOIN Highschooler AS T3 ON T1.friend_id  =  T3.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "hard", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Highschooler JOIN Friend WHERE Highschooler.name = 'terminal' AND Highschooler.name = 'terminal'", "gold": "SELECT count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Highschooler JOIN Friend WHERE Highschooler.name = 'terminal'", "gold": "SELECT count(*) FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.ID FROM Highschooler EXCEPT SELECT Friend.student_id FROM Friend", "gold": "SELECT id FROM Highschooler EXCEPT SELECT student_id FROM Friend", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.ID FROM Highschooler EXCEPT SELECT Friend.student_id FROM Friend", "gold": "SELECT id FROM Highschooler EXCEPT SELECT student_id FROM Friend", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler EXCEPT SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal'", "gold": "SELECT name FROM Highschooler EXCEPT SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler EXCEPT SELECT Highschooler.name FROM Highschooler JOIN Friend", "gold": "SELECT name FROM Highschooler EXCEPT SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Friend.friend_id FROM Friend JOIN Highschooler WHERE Highschooler.grade = 'terminal' INTERSECT SELECT Friend.friend_id FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal'", "gold": "SELECT student_id FROM Friend INTERSECT SELECT liked_id FROM Likes", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Friend.student_id FROM Friend INTERSECT SELECT Friend.student_id FROM Friend", "gold": "SELECT student_id FROM Friend INTERSECT SELECT liked_id FROM Likes", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend EXCEPT SELECT Highschooler.name FROM Highschooler JOIN Friend", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id INTERSECT SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.liked_id  =  T2.id", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal' INTERSECT SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade = 'terminal'", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id INTERSECT SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.liked_id  =  T2.id", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Count(*), Friend.student_id FROM Friend GROUP BY Friend.student_id", "gold": "SELECT student_id ,  count(*) FROM Likes GROUP BY student_id", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Friend.student_id, Count(*) FROM Friend GROUP BY Friend.student_id", "gold": "SELECT student_id ,  count(*) FROM Likes GROUP BY student_id", "predicted_parse_error": false, "hardness": "medium", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name, * FROM Likes JOIN Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT T2.name ,  count(*) FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name, Count(*) FROM Likes JOIN Highschooler WHERE Highschooler.grade = 'terminal'", "gold": "SELECT T2.name ,  count(*) FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler WHERE Highschooler.grade = (SELECT Max(*) FROM Highschooler)", "gold": "SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 1}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler ORDER BY Highschooler.grade Desc LIMIT 1", "gold": "SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}}}, {"predicted": "SELECT Highschooler.name FROM Likes JOIN Highschooler GROUP BY Likes.student_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Highschooler.name FROM Likes JOIN Highschooler GROUP BY Likes.student_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.name FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id GROUP BY T1.student_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade > 'terminal' GROUP BY Friend.student_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.grade  >  5 GROUP BY T1.student_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Highschooler.name FROM Highschooler JOIN Friend WHERE Highschooler.grade > 'terminal' AND Friend.friend_id >= 'terminal'", "gold": "SELECT T2.name FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.grade  >  5 GROUP BY T1.student_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Likes JOIN Highschooler WHERE Highschooler.name = 'terminal'", "gold": "SELECT count(*) FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Likes JOIN Highschooler WHERE Highschooler.name = 'terminal' AND Highschooler.name = 'terminal'", "gold": "SELECT count(*) FROM Likes AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id WHERE T2.name  =  \"Kyle\"", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(Highschooler.grade) FROM Highschooler JOIN Friend", "gold": "SELECT avg(grade) FROM Highschooler WHERE id IN (SELECT T1.student_id FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Avg(Highschooler.grade) FROM Highschooler JOIN Friend", "gold": "SELECT avg(grade) FROM Highschooler WHERE id IN (SELECT T1.student_id FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id)", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Min(Highschooler.grade) FROM Highschooler WHERE Highschooler.ID NOT IN (SELECT Friend.student_id FROM Friend)", "gold": "SELECT min(grade) FROM Highschooler WHERE id NOT IN (SELECT T1.student_id FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id)", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Highschooler.grade FROM Highschooler EXCEPT SELECT Highschooler.grade FROM Highschooler JOIN Friend", "gold": "SELECT min(grade) FROM Highschooler WHERE id NOT IN (SELECT T1.student_id FROM Friend AS T1 JOIN Highschooler AS T2 ON T1.student_id  =  T2.id)", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Owners.state FROM Owners UNION SELECT Owners.street FROM Owners", "gold": "SELECT state FROM Owners INTERSECT SELECT state FROM Professionals", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.state FROM Owners UNION SELECT Owners.street FROM Owners", "gold": "SELECT state FROM Owners INTERSECT SELECT state FROM Professionals", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(Dogs.age) FROM Dogs WHERE Dogs.dog_id NOT IN (SELECT Dogs.dog_id FROM Dogs)", "gold": "SELECT avg(age) FROM Dogs WHERE dog_id IN ( SELECT dog_id FROM Treatments )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 3}}}, {"predicted": "SELECT Avg(Dogs.age) FROM Dogs", "gold": "SELECT avg(age) FROM Dogs WHERE dog_id IN ( SELECT dog_id FROM Treatments )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 0}}}, {"predicted": "SELECT Owners.owner_id, Owners.last_name, Owners.home_phone FROM Owners WHERE Owners.state = 'terminal' GROUP BY Owners.owner_id HAVING Count(*) > 'terminal' UNION SELECT Owners.owner_id, Owners.last_name, Owners.home_phone FROM Owners GROUP BY Owners.owner_id HAVING Count(*) > 'terminal'", "gold": "SELECT professional_id ,  last_name ,  cell_number FROM Professionals WHERE state  =  'Indiana' UNION SELECT T1.professional_id ,  T1.last_name ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >  2", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Owners.last_name, Owners.cell_number FROM Owners WHERE Owners.state = 'terminal' GROUP BY Owners.owner_id HAVING Count(*) > 'terminal' UNION SELECT Owners.owner_id, Owners.last_name, Owners.cell_number FROM Owners WHERE Owners.state = 'terminal' GROUP BY Owners.owner_id HAVING Count(*) > 'terminal'", "gold": "SELECT professional_id ,  last_name ,  cell_number FROM Professionals WHERE state  =  'Indiana' UNION SELECT T1.professional_id ,  T1.last_name ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >  2", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 4}}}, {"predicted": "SELECT Dogs.name FROM Dogs EXCEPT SELECT Dogs.name FROM Dogs WHERE Dogs.abandoned_yn > 'terminal'", "gold": "select name from dogs where dog_id not in ( select dog_id from treatments group by dog_id having sum(cost_of_treatment)  >  1000 )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Dogs.name FROM Dogs EXCEPT SELECT Dogs.name FROM Dogs", "gold": "select name from dogs where dog_id not in ( select dog_id from treatments group by dog_id having sum(cost_of_treatment)  >  1000 )", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 1}}}, {"predicted": "SELECT Owners.first_name FROM Owners WHERE Owners.city = 'terminal' OR Owners.first_name = 'terminal'", "gold": "SELECT first_name FROM Professionals UNION SELECT first_name FROM Owners EXCEPT SELECT name FROM Dogs", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Owners.first_name FROM Owners WHERE Owners.last_name = 'terminal' OR Owners.last_name = 'terminal'", "gold": "SELECT first_name FROM Professionals UNION SELECT first_name FROM Owners EXCEPT SELECT name FROM Dogs", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}}}, {"predicted": "SELECT Owners.owner_id, Owners.email_address, Owners.email_address FROM Owners EXCEPT SELECT Owners.owner_id, Owners.email_address, Owners.email_address FROM Owners JOIN Dogs", "gold": "SELECT professional_id ,  role_code ,  email_address FROM Professionals EXCEPT SELECT T1.professional_id ,  T1.role_code ,  T1.email_address FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.owner_id, Owners.email_address, Owners.email_address FROM Owners EXCEPT SELECT Owners.owner_id, Owners.email_address, Owners.email_address FROM Owners JOIN Dogs", "gold": "SELECT professional_id ,  role_code ,  email_address FROM Professionals EXCEPT SELECT T1.professional_id ,  T1.role_code ,  T1.email_address FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.owner_id, Owners.first_name, Owners.last_name FROM Dogs JOIN Owners GROUP BY Owners.owner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T2.first_name ,  T2.last_name FROM Dogs AS T1 JOIN Owners AS T2 ON T1.owner_id  =  T2.owner_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Dogs.owner_id, Owners.first_name, Owners.last_name FROM Dogs JOIN Owners GROUP BY Dogs.owner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T2.first_name ,  T2.last_name FROM Dogs AS T1 JOIN Owners AS T2 ON T1.owner_id  =  T2.owner_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Owners.first_name, Owners.first_name FROM Owners JOIN Dogs GROUP BY Owners.owner_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.professional_id ,  T1.role_code ,  T1.first_name FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Owners.owner_id, Professionals.role_code, Owners.first_name FROM Dogs JOIN Owners GROUP BY Dogs.owner_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.professional_id ,  T1.role_code ,  T1.first_name FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Dogs.name FROM Dogs GROUP BY Dogs.name ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.breed_name FROM Breeds AS T1 JOIN Dogs AS T2 ON T1.breed_code  =  T2.breed_code GROUP BY T1.breed_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Dogs.name FROM Dogs GROUP BY Dogs.name ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.breed_name FROM Breeds AS T1 JOIN Dogs AS T2 ON T1.breed_code  =  T2.breed_code GROUP BY T1.breed_name ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Owners.last_name FROM Dogs JOIN Owners GROUP BY Owners.owner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T1.last_name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Owners.last_name FROM Dogs JOIN Owners GROUP BY Dogs.owner_id ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T1.last_name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY count(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Charges.charge_type FROM Charges GROUP BY Charges.charge_type ORDER BY Sum(Charges.charge_type) Asc LIMIT 1", "gold": "SELECT T1.treatment_type_description FROM Treatment_types AS T1 JOIN Treatments AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code GROUP BY T1.treatment_type_code ORDER BY sum(cost_of_treatment) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Treatment_Types.treatment_type_description FROM Treatment_Types JOIN Charges GROUP BY Charges.charge_type ORDER BY Sum(Treatment_Types.treatment_type_description) Asc LIMIT 1", "gold": "SELECT T1.treatment_type_description FROM Treatment_types AS T1 JOIN Treatments AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code GROUP BY T1.treatment_type_code ORDER BY sum(cost_of_treatment) ASC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Sum(Dogs.date_arrived) FROM Dogs JOIN Owners GROUP BY Dogs.owner_id ORDER BY Sum(Dogs.date_arrived) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T1.zip_code FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY sum(T3.cost_of_treatment) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Dogs.owner_id, Max(Dogs.date_arrived) FROM Dogs GROUP BY Dogs.owner_id ORDER BY Sum(Dogs.date_arrived) Desc LIMIT 1", "gold": "SELECT T1.owner_id ,  T1.zip_code FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id JOIN Treatments AS T3 ON T2.dog_id  =  T3.dog_id GROUP BY T1.owner_id ORDER BY sum(T3.cost_of_treatment) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT Owners.owner_id, Owners.home_phone FROM Owners JOIN Dogs GROUP BY Owners.owner_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.professional_id ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Charges.charge_id, Owners.cell_number FROM Charges JOIN Owners GROUP BY Charges.charge_id HAVING Count(*) >= 'terminal'", "gold": "SELECT T1.professional_id ,  T1.cell_number FROM Professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id GROUP BY T1.professional_id HAVING count(*)  >=  2", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name FROM Dogs JOIN Owners WHERE Dogs.weight < (SELECT Avg(Dogs.weight) FROM Dogs)", "gold": "SELECT DISTINCT T1.first_name ,  T1.last_name FROM Professionals AS T1 JOIN Treatments AS T2 WHERE cost_of_treatment  <  ( SELECT avg(cost_of_treatment) FROM Treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name FROM Dogs JOIN Owners WHERE Dogs.size_code < (SELECT Avg(Dogs.size_code) FROM Dogs)", "gold": "SELECT DISTINCT T1.first_name ,  T1.last_name FROM Professionals AS T1 JOIN Treatments AS T2 WHERE cost_of_treatment  <  ( SELECT avg(cost_of_treatment) FROM Treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Dogs.date_of_birth, Owners.first_name FROM Dogs JOIN Owners", "gold": "SELECT T1.date_of_treatment ,  T2.first_name FROM Treatments AS T1 JOIN Professionals AS T2 ON T1.professional_id  =  T2.professional_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.state, Owners.first_name FROM Owners", "gold": "SELECT T1.date_of_treatment ,  T2.first_name FROM Treatments AS T1 JOIN Professionals AS T2 ON T1.professional_id  =  T2.professional_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Charges.charge_type, Charges.charge_type FROM Charges", "gold": "SELECT T1.cost_of_treatment ,  T2.treatment_type_description FROM Treatments AS T1 JOIN treatment_types AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Charges.charge_amount, Charges.charge_type FROM Charges", "gold": "SELECT T1.cost_of_treatment ,  T2.treatment_type_description FROM Treatments AS T1 JOIN treatment_types AS T2 ON T1.treatment_type_code  =  T2.treatment_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name, Owners.city FROM Dogs JOIN Owners", "gold": "SELECT T1.first_name ,  T1.last_name ,  T2.size_code FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name, Dogs.gender FROM Dogs JOIN Owners", "gold": "SELECT T1.first_name ,  T1.last_name ,  T2.size_code FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Dogs.name FROM Owners JOIN Dogs", "gold": "SELECT T1.first_name ,  T2.name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Dogs.name FROM Owners JOIN Dogs", "gold": "SELECT T1.first_name ,  T2.name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Dogs.name, Dogs.date_of_birth FROM Dogs ORDER BY Dogs.date_arrived Asc LIMIT 1", "gold": "SELECT T1.name ,  T2.date_of_treatment FROM Dogs AS T1 JOIN Treatments AS T2 ON T1.dog_id  =  T2.dog_id WHERE T1.breed_code  =  ( SELECT breed_code FROM Dogs GROUP BY breed_code ORDER BY count(*) ASC LIMIT 1 )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT Dogs.name, Dogs.date_of_birth FROM Dogs ORDER BY Dogs.date_arrived Asc LIMIT 1", "gold": "SELECT T1.name ,  T2.date_of_treatment FROM Dogs AS T1 JOIN Treatments AS T2 ON T1.dog_id  =  T2.dog_id WHERE T1.breed_code  =  ( SELECT breed_code FROM Dogs GROUP BY breed_code ORDER BY count(*) ASC LIMIT 1 )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 3}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name FROM Owners WHERE Owners.state = 'terminal'", "gold": "SELECT T1.first_name ,  T2.name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id WHERE T1.state  =  'Virginia'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.first_name, Dogs.name FROM Owners JOIN Dogs WHERE Owners.state = 'terminal'", "gold": "SELECT T1.first_name ,  T2.name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id WHERE T1.state  =  'Virginia'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Dogs.date_of_birth, Dogs.date_arrived FROM Dogs", "gold": "SELECT DISTINCT T1.date_arrived ,  T1.date_departed FROM Dogs AS T1 JOIN Treatments AS T2 ON T1.dog_id  =  T2.dog_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Dogs.date_of_birth, Dogs.date_arrived FROM Dogs", "gold": "SELECT DISTINCT T1.date_arrived ,  T1.date_departed FROM Dogs AS T1 JOIN Treatments AS T2 ON T1.dog_id  =  T2.dog_id", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.last_name FROM Dogs JOIN Owners WHERE Dogs.age = (SELECT Max(Dogs.age) FROM Dogs)", "gold": "SELECT T1.last_name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id WHERE T2.age  =  ( SELECT max(age) FROM Dogs )", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Owners.last_name FROM Dogs JOIN Owners WHERE Dogs.age = (SELECT Max(Dogs.age) FROM Dogs)", "gold": "SELECT T1.last_name FROM Owners AS T1 JOIN Dogs AS T2 ON T1.owner_id  =  T2.owner_id WHERE T2.age  =  ( SELECT max(age) FROM Dogs )", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Professionals.email_address FROM Professionals WHERE Professionals.state = 'terminal' OR Professionals.state = 'terminal'", "gold": "SELECT email_address FROM Professionals WHERE state  =  'Hawaii' OR state  =  'Wisconsin'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Professionals.email_address FROM Professionals WHERE Professionals.state = 'terminal' OR Professionals.state = 'terminal'", "gold": "SELECT email_address FROM Professionals WHERE state  =  'Hawaii' OR state  =  'Wisconsin'", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Dogs.date_of_birth, Dogs.gender FROM Dogs", "gold": "SELECT date_arrived ,  date_departed FROM Dogs", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Dogs.date_of_birth, Dogs.date_arrived FROM Dogs", "gold": "SELECT date_arrived ,  date_departed FROM Dogs", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Dogs", "gold": "SELECT count(DISTINCT dog_id) FROM Treatments", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Dogs", "gold": "SELECT count(DISTINCT dog_id) FROM Treatments", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(DISTINCT Dogs.owner_id) FROM Dogs", "gold": "SELECT count(DISTINCT professional_id) FROM Treatments", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM Dogs", "gold": "SELECT count(DISTINCT professional_id) FROM Treatments", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Professionals.role_code, Professionals.home_phone, Professionals.state FROM Professionals WHERE Professionals.city LIKE 'terminal'", "gold": "SELECT role_code ,  street ,  city ,  state FROM professionals WHERE city LIKE '%West%'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Professionals.role_code, Professionals.street, Professionals.state FROM Professionals WHERE Professionals.city LIKE 'terminal'", "gold": "SELECT role_code ,  street ,  city ,  state FROM professionals WHERE city LIKE '%West%'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 4, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name, Owners.email_address FROM Owners WHERE Owners.first_name LIKE 'terminal'", "gold": "SELECT first_name ,  last_name ,  email_address FROM Owners WHERE state LIKE '%North%'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name, Owners.email_address FROM Owners WHERE Owners.first_name LIKE 'terminal'", "gold": "SELECT first_name ,  last_name ,  email_address FROM Owners WHERE state LIKE '%North%'", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM Dogs WHERE Dogs.age < (SELECT Avg(Dogs.age) FROM Dogs)", "gold": "SELECT count(*) FROM Dogs WHERE age  <  ( SELECT avg(age) FROM Dogs )", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Count(*) FROM Dogs WHERE Dogs.age < (SELECT Avg(Dogs.age) FROM Dogs)", "gold": "SELECT count(*) FROM Dogs WHERE age  <  ( SELECT avg(age) FROM Dogs )", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Dogs.date_of_birth FROM Dogs ORDER BY Dogs.date_of_birth Asc LIMIT 1", "gold": "SELECT cost_of_treatment FROM Treatments ORDER BY date_of_treatment DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Dogs.date_arrived FROM Dogs ORDER BY Dogs.date_adopted Desc LIMIT 1", "gold": "SELECT cost_of_treatment FROM Treatments ORDER BY date_of_treatment DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Dogs WHERE Dogs.dog_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "SELECT count(*) FROM Dogs WHERE dog_id NOT IN ( SELECT dog_id FROM Treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Dogs WHERE Dogs.owner_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "select count(*) from dogs where dog_id not in ( select dog_id from treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Owners WHERE Owners.owner_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "SELECT count(*) FROM Owners WHERE owner_id NOT IN ( SELECT owner_id FROM Dogs )", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Owners WHERE Owners.owner_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "SELECT count(*) FROM Owners WHERE owner_id NOT IN ( SELECT owner_id FROM Dogs )", "predicted_parse_error": false, "hardness": "extra", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Owners WHERE Owners.owner_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "SELECT count(*) FROM Professionals WHERE professional_id NOT IN ( SELECT professional_id FROM Treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Count(*) FROM Dogs WHERE Owners.owner_id NOT IN (SELECT Dogs.owner_id FROM Dogs)", "gold": "SELECT count(*) FROM Professionals WHERE professional_id NOT IN ( SELECT professional_id FROM Treatments )", "predicted_parse_error": false, "hardness": "extra", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT Dogs.name, Dogs.age, Dogs.weight FROM Dogs", "gold": "SELECT name ,  age ,  weight FROM Dogs WHERE abandoned_yn  =  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Dogs.name, Dogs.age, Dogs.weight FROM Dogs UNION SELECT Dogs.name, Dogs.weight, Dogs.weight FROM Dogs", "gold": "SELECT name ,  age ,  weight FROM Dogs WHERE abandoned_yn  =  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Avg(Dogs.age) FROM Dogs", "gold": "SELECT avg(age) FROM Dogs", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Avg(Dogs.age) FROM Dogs", "gold": "SELECT avg(age) FROM Dogs", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Dogs.age FROM Dogs ORDER BY Dogs.age Desc LIMIT 1", "gold": "SELECT max(age) FROM Dogs", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 3}}}, {"predicted": "SELECT Dogs.age FROM Dogs ORDER BY Dogs.age Desc LIMIT 1", "gold": "SELECT max(age) FROM Dogs", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 3}}}, {"predicted": "SELECT Charges.charge_type, Count(*) FROM Charges GROUP BY Charges.charge_type", "gold": "SELECT charge_type ,  charge_amount FROM Charges", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Charges.charge_type, Count(*) FROM Charges GROUP BY Charges.charge_type", "gold": "SELECT charge_type ,  charge_amount FROM Charges", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}}}, {"predicted": "SELECT Charges.charge_type FROM Charges GROUP BY Charges.charge_type ORDER BY Max(Charges.charge_amount) Desc LIMIT 1", "gold": "SELECT max(charge_amount) FROM Charges", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 4}}}, {"predicted": "SELECT Charges.charge_type FROM Charges GROUP BY Charges.charge_type ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT max(charge_amount) FROM Charges", "predicted_parse_error": false, "hardness": "easy", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 0, "pred_total": 4}}}, {"predicted": "SELECT Owners.email_address, Owners.cell_number, Owners.home_phone FROM Owners", "gold": "SELECT email_address ,  cell_number ,  home_phone FROM professionals", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.email_address, Owners.cell_number, Owners.home_phone FROM Owners", "gold": "SELECT email_address ,  cell_number ,  home_phone FROM professionals", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 3, "pred_total": 3}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Charges.charge_type FROM Charges", "gold": "SELECT DISTINCT breed_code ,  size_code FROM dogs", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT Charges.charge_type, Charges.charge_amount FROM Charges", "gold": "SELECT DISTINCT breed_code ,  size_code FROM dogs", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Owners.last_name FROM Dogs JOIN Owners", "gold": "SELECT DISTINCT T1.first_name ,  T3.treatment_type_description FROM professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id JOIN Treatment_types AS T3 ON T2.treatment_type_code  =  T3.treatment_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Owners.first_name, Treatment_Types.treatment_type_description FROM Owners JOIN Treatment_Types", "gold": "SELECT DISTINCT T1.first_name ,  T3.treatment_type_description FROM professionals AS T1 JOIN Treatments AS T2 ON T1.professional_id  =  T2.professional_id JOIN Treatment_types AS T3 ON T2.treatment_type_code  =  T3.treatment_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM singer", "gold": "SELECT count(*) FROM singer", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Count(*) FROM singer", "gold": "SELECT count(*) FROM singer", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT singer.Name FROM singer ORDER BY singer.Net_Worth_Millions Asc", "gold": "SELECT Name FROM singer ORDER BY Net_Worth_Millions ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name FROM singer ORDER BY singer.Net_Worth_Millions Asc", "gold": "SELECT Name FROM singer ORDER BY Net_Worth_Millions ASC", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Birth_Year, singer.Citizenship FROM singer", "gold": "SELECT Birth_Year ,  Citizenship FROM singer", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT singer.Birth_Year, singer.Citizenship FROM singer", "gold": "SELECT Birth_Year ,  Citizenship FROM singer", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Citizenship != 'terminal'", "gold": "SELECT Name FROM singer WHERE Citizenship != \"France\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Citizenship != 'terminal'", "gold": "SELECT Name FROM singer WHERE Citizenship != \"France\"", "predicted_parse_error": false, "hardness": "easy", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Birth_Year = 'terminal' OR singer.Birth_Year = 'terminal'", "gold": "SELECT Name FROM singer WHERE Birth_Year  =  1948 OR Birth_Year  =  1949", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Birth_Year = 'terminal' OR singer.Birth_Year = 'terminal'", "gold": "SELECT Name FROM singer WHERE Birth_Year  =  1948 OR Birth_Year  =  1949", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name, singer.Name FROM singer ORDER BY singer.Net_Worth_Millions Desc LIMIT 1", "gold": "SELECT Name FROM singer ORDER BY Net_Worth_Millions DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Name, singer.Name FROM singer JOIN song ORDER BY song.Sales Desc LIMIT 1", "gold": "SELECT Name FROM singer ORDER BY Net_Worth_Millions DESC LIMIT 1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Citizenship, Count(*) FROM singer GROUP BY singer.Citizenship", "gold": "SELECT Citizenship ,  COUNT(*) FROM singer GROUP BY Citizenship", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Citizenship, Count(*) FROM singer GROUP BY singer.Citizenship", "gold": "SELECT Citizenship ,  COUNT(*) FROM singer GROUP BY Citizenship", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Citizenship FROM singer GROUP BY singer.Citizenship ORDER BY Count(*) Desc LIMIT 1", "gold": "SELECT Citizenship FROM singer GROUP BY Citizenship ORDER BY COUNT(*) DESC LIMIT 1", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT singer.Birth_Year FROM singer GROUP BY singer.Birth_Year ORDER BY Count(*) Desc LIMIT 1", "gold": "select citizenship from singer group by citizenship order by count(*) desc limit 1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 4, "pred_total": 4}}}, {"predicted": "SELECT singer.Citizenship, Max(song.Sales) FROM singer JOIN song GROUP BY singer.Citizenship", "gold": "SELECT Citizenship ,  max(Net_Worth_Millions) FROM singer GROUP BY Citizenship", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Max(song.Sales), singer.Citizenship FROM singer JOIN song GROUP BY singer.Citizenship", "gold": "SELECT Citizenship ,  max(Net_Worth_Millions) FROM singer GROUP BY Citizenship", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT song.Title, singer.Name FROM singer JOIN song", "gold": "SELECT T2.Title ,  T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT song.Title, singer.Name FROM singer JOIN song", "gold": "SELECT T2.Title ,  T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT DISTINCT singer.Name FROM singer JOIN song WHERE song.Sales > 'terminal'", "gold": "SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT DISTINCT singer.Name FROM singer JOIN song WHERE song.Sales > 'terminal'", "gold": "SELECT DISTINCT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID WHERE T2.Sales  >  300000", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer JOIN song GROUP BY song.Singer_ID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Name HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name FROM singer JOIN song GROUP BY song.Singer_ID HAVING Count(*) > 'terminal'", "gold": "SELECT T1.Name FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Name HAVING COUNT(*)  >  1", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Name, Sum(song.Sales) FROM singer JOIN song GROUP BY song.Singer_ID", "gold": "SELECT T1.Name ,  sum(T2.Sales) FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Name", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Sum(song.Sales), singer.Name FROM singer JOIN song GROUP BY singer.Name", "gold": "SELECT T1.Name ,  sum(T2.Sales) FROM singer AS T1 JOIN song AS T2 ON T1.Singer_ID  =  T2.Singer_ID GROUP BY T1.Name", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Singer_ID NOT IN (SELECT song.Singer_ID FROM song)", "gold": "SELECT Name FROM singer WHERE Singer_ID NOT IN (SELECT Singer_ID FROM song)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Name FROM singer WHERE singer.Singer_ID NOT IN (SELECT song.Singer_ID FROM song)", "gold": "SELECT Name FROM singer WHERE Singer_ID NOT IN (SELECT Singer_ID FROM song)", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 3, "pred_total": 3}}}, {"predicted": "SELECT singer.Citizenship FROM singer WHERE singer.Birth_Year < 'terminal' INTERSECT SELECT singer.Citizenship FROM singer WHERE singer.Birth_Year > 'terminal'", "gold": "SELECT Citizenship FROM singer WHERE Birth_Year  <  1945 INTERSECT SELECT Citizenship FROM singer WHERE Birth_Year  >  1955", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT singer.Citizenship FROM singer WHERE singer.Birth_Year < 'terminal' INTERSECT SELECT singer.Citizenship FROM singer WHERE singer.Birth_Year > 'terminal'", "gold": "SELECT Citizenship FROM singer WHERE Birth_Year  <  1945 INTERSECT SELECT Citizenship FROM singer WHERE Birth_Year  >  1955", "predicted_parse_error": false, "hardness": "hard", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 2, "pred_total": 2}}}, {"predicted": "SELECT Count(*) FROM Properties", "gold": "SELECT count(*) FROM Other_Available_Features", "predicted_parse_error": false, "hardness": "easy", "exact": false, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}}}, {"predicted": "SELECT Ref_Feature_Types.feature_type_name FROM Other_Available_Features JOIN Ref_Feature_Types WHERE Other_Available_Features.feature_name = 'terminal'", "gold": "SELECT T2.feature_type_name FROM Other_Available_Features AS T1 JOIN Ref_Feature_Types AS T2 ON T1.feature_type_code  =  T2.feature_type_code WHERE T1.feature_name  =  \"AirCon\"", "predicted_parse_error": false, "hardness": "medium", "exact": true, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}}}, {"predicted": "SELECT Other_Available_Features.feature_description FROM Other_Available_Features", "gold": "SELECT T2.property_type_description FROM Properties AS T1 JOIN Ref_Property_Types AS T2 ON T1.property_type_code  =  T2.property_type_code GROUP BY T1.property_type_code", "predicted_parse_error": false, "hardness": "medium", "exact": 0, "partial": {"select": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 1}, "where": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "where(no OP)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group(no Having)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "group": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "IUEN": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}}}, {"predicted": "SELECT Properties.property_name FROM Properties WHERE Properties.room_count > 'terminal' OR Properties.room_count > 'terminal'", "gold": "SELECT property_name FROM Properties WHERE property_type_code  =  \"House\" UNION SELECT property_name FROM Properties WHERE property_type_code  =  \"Apartment\" AND room_count  >  1", "predicted_parse_error": false, "hardness": "hard", "exact": 0, "partial": {"select": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "select(no AGG)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 1, "pred_total": 1}, "where": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "where(no OP)": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 2}, "group(no Having)": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "group": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "order": {"acc": 1, "rec": 1, "f1": 1, "label_total": 0, "pred_total": 0}, "and/or": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "IUEN": {"acc": 0, "rec": 0, "f1": 0, "label_total": 1, "pred_total": 0}, "keywords": {"acc": 0, "rec": 0, "f1": 0, "label_total": 2, "pred_total": 2}}}], "total_scores": {"easy": {"count": 248, "partial": {"select": {"acc": 0.8064516129032258, "rec": 0.8064516129032258, "f1": 0.8064516129032258, "acc_count": 248, "rec_count": 248}, "select(no AGG)": {"acc": 0.8185483870967742, "rec": 0.8185483870967742, "f1": 0.8185483870967742, "acc_count": 248, "rec_count": 248}, "where": {"acc": 0.6697247706422018, "rec": 0.6759259259259259, "f1": 0.6728110599078341, "acc_count": 109, "rec_count": 108}, "where(no OP)": {"acc": 0.6788990825688074, "rec": 0.6851851851851852, "f1": 0.6820276497695853, "acc_count": 109, "rec_count": 108}, "group(no Having)": {"acc": 0.6666666666666666, "rec": 0.8, "f1": 0.7272727272727272, "acc_count": 24, "rec_count": 20}, "group": {"acc": 0.6666666666666666, "rec": 0.8, "f1": 0.7272727272727272, "acc_count": 24, "rec_count": 20}, "order": {"acc": 0.6153846153846154, "rec": 0.7272727272727273, "f1": 0.6666666666666667, "acc_count": 26, "rec_count": 22}, "and/or": {"acc": 1.0, "rec": 0.9959677419354839, "f1": 0.997979797979798, "acc_count": 247, "rec_count": 248}, "IUEN": {"acc": 0.0, "rec": 0, "f1": 1, "acc_count": 2, "rec_count": 0}, "keywords": {"acc": 0.8701298701298701, "rec": 0.8933333333333333, "f1": 0.881578947368421, "acc_count": 154, "rec_count": 150}}, "exact": 0.5645161290322581, "exec": 0}, "medium": {"count": 446, "partial": {"select": {"acc": 0.6621621621621622, "rec": 0.6591928251121076, "f1": 0.6606741573033708, "acc_count": 444, "rec_count": 446}, "select(no AGG)": {"acc": 0.6779279279279279, "rec": 0.6748878923766816, "f1": 0.6764044943820224, "acc_count": 444, "rec_count": 446}, "where": {"acc": 0.6613756613756614, "rec": 0.6868131868131868, "f1": 0.6738544474393531, "acc_count": 189, "rec_count": 182}, "where(no OP)": {"acc": 0.671957671957672, "rec": 0.6978021978021978, "f1": 0.6846361185983828, "acc_count": 189, "rec_count": 182}, "group(no Having)": {"acc": 0.6504065040650406, "rec": 0.6015037593984962, "f1": 0.625, "acc_count": 123, "rec_count": 133}, "group": {"acc": 0.5934959349593496, "rec": 0.5488721804511278, "f1": 0.5703125, "acc_count": 123, "rec_count": 133}, "order": {"acc": 0.5866666666666667, "rec": 0.5866666666666667, "f1": 0.5866666666666667, "acc_count": 75, "rec_count": 75}, "and/or": {"acc": 0.9661399548532731, "rec": 0.9930394431554525, "f1": 0.9794050343249427, "acc_count": 443, "rec_count": 431}, "IUEN": {"acc": 0.0, "rec": 0, "f1": 1, "acc_count": 1, "rec_count": 0}, "keywords": {"acc": 0.8940217391304348, "rec": 0.8703703703703703, "f1": 0.8820375335120644, "acc_count": 368, "rec_count": 378}}, "exact": 0.4282511210762332, "exec": 0}, "hard": {"count": 174, "partial": {"select": {"acc": 0.8103448275862069, "rec": 0.8103448275862069, "f1": 0.8103448275862069, "acc_count": 174, "rec_count": 174}, "select(no AGG)": {"acc": 0.8103448275862069, "rec": 0.8103448275862069, "f1": 0.8103448275862069, "acc_count": 174, "rec_count": 174}, "where": {"acc": 0.5185185185185185, "rec": 0.44680851063829785, "f1": 0.48, "acc_count": 81, "rec_count": 94}, "where(no OP)": {"acc": 0.5679012345679012, "rec": 0.48936170212765956, "f1": 0.5257142857142857, "acc_count": 81, "rec_count": 94}, "group(no Having)": {"acc": 0.7, "rec": 0.717948717948718, "f1": 0.708860759493671, "acc_count": 40, "rec_count": 39}, "group": {"acc": 0.7, "rec": 0.717948717948718, "f1": 0.708860759493671, "acc_count": 40, "rec_count": 39}, "order": {"acc": 0.7547169811320755, "rec": 0.7272727272727273, "f1": 0.7407407407407407, "acc_count": 53, "rec_count": 55}, "and/or": {"acc": 0.9529411764705882, "rec": 0.9759036144578314, "f1": 0.9642857142857143, "acc_count": 170, "rec_count": 166}, "IUEN": {"acc": 0.3695652173913043, "rec": 0.40476190476190477, "f1": 0.38636363636363635, "acc_count": 46, "rec_count": 42}, "keywords": {"acc": 0.7602339181286549, "rec": 0.7471264367816092, "f1": 0.7536231884057971, "acc_count": 171, "rec_count": 174}}, "exact": 0.3735632183908046, "exec": 0}, "extra": {"count": 166, "partial": {"select": {"acc": 0.6325301204819277, "rec": 0.6325301204819277, "f1": 0.6325301204819277, "acc_count": 166, "rec_count": 166}, "select(no AGG)": {"acc": 0.6385542168674698, "rec": 0.6385542168674698, "f1": 0.6385542168674698, "acc_count": 166, "rec_count": 166}, "where": {"acc": 0.34523809523809523, "rec": 0.30851063829787234, "f1": 0.3258426966292135, "acc_count": 84, "rec_count": 94}, "where(no OP)": {"acc": 0.44047619047619047, "rec": 0.39361702127659576, "f1": 0.4157303370786517, "acc_count": 84, "rec_count": 94}, "group(no Having)": {"acc": 0.6351351351351351, "rec": 0.5949367088607594, "f1": 0.6143790849673201, "acc_count": 74, "rec_count": 79}, "group": {"acc": 0.6216216216216216, "rec": 0.5822784810126582, "f1": 0.6013071895424836, "acc_count": 74, "rec_count": 79}, "order": {"acc": 0.7763157894736842, "rec": 0.7468354430379747, "f1": 0.7612903225806452, "acc_count": 76, "rec_count": 79}, "and/or": {"acc": 0.901840490797546, "rec": 0.9671052631578947, "f1": 0.9333333333333332, "acc_count": 163, "rec_count": 152}, "IUEN": {"acc": 0.16666666666666666, "rec": 0.14705882352941177, "f1": 0.15625, "acc_count": 30, "rec_count": 34}, "keywords": {"acc": 0.7515151515151515, "rec": 0.7469879518072289, "f1": 0.7492447129909365, "acc_count": 165, "rec_count": 166}}, "exact": 0.21686746987951808, "exec": 0}, "all": {"count": 1034, "partial": {"select": {"acc": 0.7170542635658915, "rec": 0.7156673114119922, "f1": 0.7163601161665052, "acc_count": 1032, "rec_count": 1034}, "select(no AGG)": {"acc": 0.7277131782945736, "rec": 0.7263056092843327, "f1": 0.7270087124878993, "acc_count": 1032, "rec_count": 1034}, "where": {"acc": 0.5809935205183585, "rec": 0.5627615062761506, "f1": 0.5717321997874601, "acc_count": 463, "rec_count": 478}, "where(no OP)": {"acc": 0.6133909287257019, "rec": 0.5941422594142259, "f1": 0.6036131774707758, "acc_count": 463, "rec_count": 478}, "group(no Having)": {"acc": 0.6551724137931034, "rec": 0.6309963099630996, "f1": 0.6428571428571429, "acc_count": 261, "rec_count": 271}, "group": {"acc": 0.6245210727969349, "rec": 0.6014760147601476, "f1": 0.6127819548872181, "acc_count": 261, "rec_count": 271}, "order": {"acc": 0.691304347826087, "rec": 0.6883116883116883, "f1": 0.6898047722342733, "acc_count": 230, "rec_count": 231}, "and/or": {"acc": 0.9618768328445748, "rec": 0.9869608826479438, "f1": 0.9742574257425742, "acc_count": 1023, "rec_count": 997}, "IUEN": {"acc": 0.27848101265822783, "rec": 0.2894736842105263, "f1": 0.2838709677419355, "acc_count": 79, "rec_count": 76}, "keywords": {"acc": 0.8356643356643356, "rec": 0.826036866359447, "f1": 0.8308227114716107, "acc_count": 858, "rec_count": 868}}, "exact": 0.41779497098646035, "exec": 0}}}