test with different data

dashboard_utils/bubbles.py

@@ -1,4 +1,5 @@
 import datetime
+import json
 from concurrent.futures import as_completed
 from urllib import parse
 
@@ -13,19 +14,25 @@ WANDB_REPO = "learning-at-home/Worker_logs"
 
 @simple_time_tracker(_log)
 def get_new_bubble_data():
-    serialized_data_points, latest_timestamp = get_serialized_data_points()
+    # serialized_data_points, latest_timestamp = get_serialized_data_points()
+    serialized_data_points, latest_timestamp = None, None
     serialized_data = get_serialized_data(serialized_data_points, latest_timestamp)
-    profiles = get_profiles(serialized_data_points)
+
+    usernames = []
+    for item in serialized_data["points"][0]:
+        usernames.append(item["profileId"])
+
+    profiles = get_profiles(usernames)
 
     return serialized_data, profiles
 
 
 @simple_time_tracker(_log)
-def get_profiles(serialized_data_points):
+def get_profiles(usernames):
     profiles = []
     with FuturesSession() as session:
         futures = []
-        for username in serialized_data_points.keys():
+        for username in usernames:
             future = session.get(URL_QUICKSEARCH + parse.urlencode({"type": "user", "q": username}))
             future.username = username
             futures.append(future)
@@ -100,30 +107,36 @@ def get_serialized_data_points():
 
 @simple_time_tracker(_log)
 def get_serialized_data(serialized_data_points, latest_timestamp):
-    serialized_data_points_v2 = []
-    max_velocity = 1
-    for run_name, serialized_data_point in serialized_data_points.items():
-        activeRuns = []
-        loss = 0
-        runtime = 0
-        batches = 0
-        velocity = 0
-        for run in serialized_data_point["Runs"]:
-            if run["date"] == latest_timestamp:
-                run["date"] = run["date"].isoformat()
-                activeRuns.append(run)
-                loss += run["loss"]
-                velocity += run["velocity"]
-            loss = loss / len(activeRuns) if activeRuns else 0
-            runtime += run["runtime"]
-            batches += run["batches"]
-        new_item = {
-            "date": latest_timestamp.isoformat(),
-            "profileId": run_name,
-            "batches": batches,
-            "runtime": runtime,
-            "activeRuns": activeRuns,
-        }
-        serialized_data_points_v2.append(new_item)
-    serialized_data = {"points": [serialized_data_points_v2], "maxVelocity": max_velocity}
+    # serialized_data_points_v2 = []
+    # max_velocity = 1
+    # for run_name, serialized_data_point in serialized_data_points.items():
+    #     activeRuns = []
+    #     loss = 0
+    #     runtime = 0
+    #     batches = 0
+    #     velocity = 0
+    #     for run in serialized_data_point["Runs"]:
+    #         if run["date"] == latest_timestamp:
+    #             run["date"] = run["date"].isoformat()
+    #             activeRuns.append(run)
+    #             loss += run["loss"]
+    #             velocity += run["velocity"]
+    #         loss = loss / len(activeRuns) if activeRuns else 0
+    #         runtime += run["runtime"]
+    #         batches += run["batches"]
+    #     new_item = {
+    #         "date": latest_timestamp.isoformat(),
+    #         "profileId": run_name,
+    #         "batches": batches,
+    #         "runtime": runtime,
+    #         "activeRuns": activeRuns,
+    #     }
+    #     serialized_data_points_v2.append(new_item)
+    # serialized_data = {"points": [serialized_data_points_v2], "maxVelocity": max_velocity}
+    with open(
+        "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+        "serializaledata_V2.json",
+        "r",
+    ) as f:
+        serialized_data = json.load(f)
     return serialized_data

perso/change_data.py

@@ -0,0 +1,19 @@
+import json
+import random
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata.json",
+    "r",
+) as f:
+    serialized_data = json.load(f)
+
+serialized_data_v2 = serialized_data
+serialized_data_v2["points"] = [[item for item in serialized_data["points"][-1] if random.random() > 0.8]]
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata_V2.json",
+    "w",
+) as f:
+    f.write(json.dumps(serialized_data_v2))

perso/get_usernames.py

@@ -0,0 +1,14 @@
+import json
+
+with open(
+    "/mnt/storage/Documents/hugging_face/colaborative_hub_training/demo_neurips/training-transformers-together-dashboard/data/"
+    "serializaledata_V2.json",
+    "r",
+) as f:
+    serialized_data = json.load(f)
+
+usernames = []
+for item in serialized_data["points"][0]:
+    usernames.append(item["profileId"])
+
+print(usernames)