diff --git "a/bitcoinforum/3_training/old/bert/bert.ipynb" "b/bitcoinforum/3_training/old/bert/bert.ipynb" new file mode 100644--- /dev/null +++ "b/bitcoinforum/3_training/old/bert/bert.ipynb" @@ -0,0 +1,1035 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "cwd = os.getcwd()\n", + "kaggle = cwd == \"/kaggle/working\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import XLMRobertaTokenizerFast\n", + "import torch\n", + "\n", + "tokenizer = XLMRobertaTokenizerFast.from_pretrained('xlm-roberta-base')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2860, 3) (101, 3)\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import json\n", + "\n", + "# Load the dataset\n", + "df = pd.read_csv('data/dataset.csv')\n", + "# df = pd.read_csv('data/dataset_old.csv')\n", + "from sklearn.model_selection import train_test_split\n", + "df, val = train_test_split(df, test_size=0.034, random_state=42)\n", + "print(df.shape, val.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Token indices sequence length is longer than the specified maximum sequence length for this model (669 > 512). Running this sequence through the model will result in indexing errors\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgW0lEQVR4nO3df2yV5f3/8deRtodS2zNKRw9nVOhcmZstRIpDiArjR7WjomERHYRhxozIj9kPEIQRIy5bS1gEtjCZLghMxmo+CTgTHKNEqBJg1gIT6sYwFiizpZOV0wK1reX6/LEv99fT8sNi8bxPeT6Sk9D7vnrOdfU+x/P07jk9PuecEwAAgCE3RXsCAAAA7REoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMCcu2hO4FhcuXNBHH32k5ORk+Xy+aE8HAAB8Ds45NTY2KhQK6aabrnyOJCYD5aOPPlJGRka0pwEAAK5BdXW1+vfvf8UxMRkoycnJkv67wJSUlCjPBgAAfB4NDQ3KyMjwnsevJCYD5eKvdVJSUggUAABizOd5eQYvkgUAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMiYv2BACgOxi4aGtUbvfYsglRuV3geuMMCgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGDOFwqU4uJi+Xw+FRYWetucc1q6dKlCoZASExM1evRoVVZWRnxfc3Oz5s6dq7S0NCUlJWnixIk6efLkF5kKAADoRq45UMrLy/XSSy9p8ODBEduXL1+uFStWaPXq1SovL1cwGNT48ePV2NjojSksLNSWLVtUUlKi3bt36+zZsyooKFBbW9u1rwQAAHQb1xQoZ8+e1dSpU/W73/1OvXv39rY757Rq1SotWbJEkyZNUnZ2tjZs2KDz589r06ZNkqRwOKy1a9fq+eef17hx43THHXdo48aNOnTokHbs2NE1qwIAADHtmgJl9uzZmjBhgsaNGxexvaqqSrW1tcrLy/O2+f1+jRo1Snv27JEkVVRUqLW1NWJMKBRSdna2N6a95uZmNTQ0RFwAAED3FdfZbygpKdH+/ftVXl7eYV9tba0kKT09PWJ7enq6jh8/7o1JSEiIOPNycczF72+vuLhYzz33XGenCgAAYlSnzqBUV1frqaee0saNG9WzZ8/LjvP5fBFfO+c6bGvvSmMWL16scDjsXaqrqzszbQAAEGM6FSgVFRWqq6tTbm6u4uLiFBcXp7KyMv36179WXFycd+ak/ZmQuro6b18wGFRLS4vq6+svO6Y9v9+vlJSUiAsAAOi+OhUoY8eO1aFDh3Tw4EHvMmzYME2dOlUHDx7U17/+dQWDQZWWlnrf09LSorKyMo0cOVKSlJubq/j4+IgxNTU1Onz4sDcGAADc2Dr1GpTk5GRlZ2dHbEtKSlKfPn287YWFhSoqKlJWVpaysrJUVFSkXr16acqUKZKkQCCgGTNmaP78+erTp49SU1O1YMEC5eTkdHjR7Y1m4KKtUbndY8smROV2AQC4nE6/SPZqFi5cqKamJs2aNUv19fUaPny4tm/fruTkZG/MypUrFRcXp8mTJ6upqUljx47V+vXr1aNHj66eDgAAiEE+55yL9iQ6q6GhQYFAQOFwuFu9HoUzKEDs4vELXF1nnr/5LB4AAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDlx0Z6ARQMXbY32FAAAuKFxBgUAAJhDoAAAAHMIFAAAYA6BAgAAzCFQAACAOQQKAAAwh7cZI2pvqz62bEJUbhcAYB9nUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5nQqUNasWaPBgwcrJSVFKSkpGjFihP785z97+51zWrp0qUKhkBITEzV69GhVVlZGXEdzc7Pmzp2rtLQ0JSUlaeLEiTp58mTXrAYAAHQLnQqU/v37a9myZXr33Xf17rvvasyYMXrwwQe9CFm+fLlWrFih1atXq7y8XMFgUOPHj1djY6N3HYWFhdqyZYtKSkq0e/dunT17VgUFBWpra+valQEAgJjVqUB54IEH9L3vfU+DBg3SoEGD9Itf/EI333yz9u3bJ+ecVq1apSVLlmjSpEnKzs7Whg0bdP78eW3atEmSFA6HtXbtWj3//PMaN26c7rjjDm3cuFGHDh3Sjh07rssCAQBA7Lnm16C0tbWppKRE586d04gRI1RVVaXa2lrl5eV5Y/x+v0aNGqU9e/ZIkioqKtTa2hoxJhQKKTs72xtzKc3NzWpoaIi4AACA7qvTgXLo0CHdfPPN8vv9mjlzprZs2aJvf/vbqq2tlSSlp6dHjE9PT/f21dbWKiEhQb17977smEspLi5WIBDwLhkZGZ2dNgAAiCGdDpRvfvObOnjwoPbt26cnn3xS06dP1/vvv+/t9/l8EeOdcx22tXe1MYsXL1Y4HPYu1dXVnZ02AACIIZ0OlISEBH3jG9/QsGHDVFxcrCFDhuhXv/qVgsGgJHU4E1JXV+edVQkGg2ppaVF9ff1lx1yK3+/33jl08QIAALqvL/x3UJxzam5uVmZmpoLBoEpLS719LS0tKisr08iRIyVJubm5io+PjxhTU1Ojw4cPe2MAAADiOjP4pz/9qfLz85WRkaHGxkaVlJRo165d2rZtm3w+nwoLC1VUVKSsrCxlZWWpqKhIvXr10pQpUyRJgUBAM2bM0Pz589WnTx+lpqZqwYIFysnJ0bhx467LAgEAQOzpVKCcOnVK06ZNU01NjQKBgAYPHqxt27Zp/PjxkqSFCxeqqalJs2bNUn19vYYPH67t27crOTnZu46VK1cqLi5OkydPVlNTk8aOHav169erR48eXbsyAAAQs3zOORftSXRWQ0ODAoGAwuHwdXk9ysBFW7v8OmHHsWUToj0FdEPR+u8G92fEks48f/NZPAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgTly0JwB82QYu2hqV2z22bEJUbhcAYhFnUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMKdTgVJcXKw777xTycnJ6tu3rx566CEdOXIkYoxzTkuXLlUoFFJiYqJGjx6tysrKiDHNzc2aO3eu0tLSlJSUpIkTJ+rkyZNffDUAAKBb6FSglJWVafbs2dq3b59KS0v16aefKi8vT+fOnfPGLF++XCtWrNDq1atVXl6uYDCo8ePHq7Gx0RtTWFioLVu2qKSkRLt379bZs2dVUFCgtra2rlsZAACIWZ36sMBt27ZFfL1u3Tr17dtXFRUVuvfee+Wc06pVq7RkyRJNmjRJkrRhwwalp6dr06ZNeuKJJxQOh7V27Vq98sorGjdunCRp48aNysjI0I4dO3Tfffd10dIAAECs+kKvQQmHw5Kk1NRUSVJVVZVqa2uVl5fnjfH7/Ro1apT27NkjSaqoqFBra2vEmFAopOzsbG9Me83NzWpoaIi4AACA7uuaA8U5p3nz5unuu+9Wdna2JKm2tlaSlJ6eHjE2PT3d21dbW6uEhAT17t37smPaKy4uViAQ8C4ZGRnXOm0AABADrjlQ5syZo/fee09//OMfO+zz+XwRXzvnOmxr70pjFi9erHA47F2qq6uvddoAACAGXFOgzJ07V6+//rp27typ/v37e9uDwaAkdTgTUldX551VCQaDamlpUX19/WXHtOf3+5WSkhJxAQAA3VenAsU5pzlz5mjz5s168803lZmZGbE/MzNTwWBQpaWl3raWlhaVlZVp5MiRkqTc3FzFx8dHjKmpqdHhw4e9MQAA4MbWqXfxzJ49W5s2bdKf/vQnJScne2dKAoGAEhMT5fP5VFhYqKKiImVlZSkrK0tFRUXq1auXpkyZ4o2dMWOG5s+frz59+ig1NVULFixQTk6O964eAABwY+tUoKxZs0aSNHr06Ijt69at02OPPSZJWrhwoZqamjRr1izV19dr+PDh2r59u5KTk73xK1euVFxcnCZPnqympiaNHTtW69evV48ePb7YagAAQLfgc865aE+isxoaGhQIBBQOh6/L61EGLtra5dcJHFs2IdpTwHUUrf9ucL9CLOnM8zefxQMAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzOvWH2gAAtkTz7zbxN1hwPXEGBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5fJox8CWJ1qfO8omzAGIRZ1AAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwBw+LBDo5viQQgCxiDMoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMCcu2hMAgK40cNHWaE8BQBfgDAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDm8zBnBd8HZfAF8EZ1AAAIA5BAoAADCHQAEAAOYQKAAAwBwCBQAAmEOgAAAAcwgUAABgDoECAADMIVAAAIA5BAoAADCHQAEAAOYQKAAAwJxOB8pbb72lBx54QKFQSD6fT6+99lrEfuecli5dqlAopMTERI0ePVqVlZURY5qbmzV37lylpaUpKSlJEydO1MmTJ7/QQgAAQPfR6UA5d+6chgwZotWrV19y//Lly7VixQqtXr1a5eXlCgaDGj9+vBobG70xhYWF2rJli0pKSrR7926dPXtWBQUFamtru/aVAACAbiOus9+Qn5+v/Pz8S+5zzmnVqlVasmSJJk2aJEnasGGD0tPTtWnTJj3xxBMKh8Nau3atXnnlFY0bN06StHHjRmVkZGjHjh267777vsByAABAd9Clr0GpqqpSbW2t8vLyvG1+v1+jRo3Snj17JEkVFRVqbW2NGBMKhZSdne2Naa+5uVkNDQ0RFwAA0H11aaDU1tZKktLT0yO2p6ene/tqa2uVkJCg3r17X3ZMe8XFxQoEAt4lIyOjK6cNAACMuS7v4vH5fBFfO+c6bGvvSmMWL16scDjsXaqrq7tsrgAAwJ4uDZRgMChJHc6E1NXVeWdVgsGgWlpaVF9ff9kx7fn9fqWkpERcAABA99WlgZKZmalgMKjS0lJvW0tLi8rKyjRy5EhJUm5uruLj4yPG1NTU6PDhw94YAABwY+v0u3jOnj2rDz74wPu6qqpKBw8eVGpqqm655RYVFhaqqKhIWVlZysrKUlFRkXr16qUpU6ZIkgKBgGbMmKH58+erT58+Sk1N1YIFC5STk+O9qwcAANzYOh0o7777rr773e96X8+bN0+SNH36dK1fv14LFy5UU1OTZs2apfr6eg0fPlzbt29XcnKy9z0rV65UXFycJk+erKamJo0dO1br169Xjx49umBJAAAg1vmccy7ak+ishoYGBQIBhcPh6/J6lIGLtnb5dQJAd3Ns2YRoTwExpjPP33wWDwAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMwhUAAAgDkECgAAMIdAAQAA5hAoAADAHAIFAACYQ6AAAABzCBQAAGAOgQIAAMyJi/YEAACxaeCirVG53WPLJkTldvHl4gwKAAAwh0ABAADmECgAAMAcAgUAAJhDoAAAAHMIFAAAYA6BAgAAzCFQAACAOQQKAAAwh0ABAADmECgAAMAcAgUAAJhDoAAAAHMIFAAAYA6BAgAAzCFQAACAOQQKAAAwh0ABAADmECgAAMAcAgUAAJgTF+0JAADQGQMXbY3K7R5bNiEqt3uj4gwKAAAwJ6qB8sILLygzM1M9e/ZUbm6u3n777WhOBwAAGBG1QHn11VdVWFioJUuW6MCBA7rnnnuUn5+vEydORGtKAADACJ9zzkXjhocPH66hQ4dqzZo13rZvfetbeuihh1RcXHzF721oaFAgEFA4HFZKSkqXzy1av98EAKC97vTal848f0flRbItLS2qqKjQokWLIrbn5eVpz549HcY3NzerubnZ+zocDkv670KvhwvN56/L9QIA0Fm3/M//RuV2Dz93X5df58Xn7c9zbiQqgfLxxx+rra1N6enpEdvT09NVW1vbYXxxcbGee+65DtszMjKu2xwBALiRBVZdv+tubGxUIBC44piovs3Y5/NFfO2c67BNkhYvXqx58+Z5X1+4cEH/+c9/1KdPn0uO/zwaGhqUkZGh6urq6/JromhjfbGvu6+R9cW+7r5G1tf1nHNqbGxUKBS66tioBEpaWpp69OjR4WxJXV1dh7MqkuT3++X3+yO2feUrX+mSuaSkpHTLO95FrC/2dfc1sr7Y193XyPq61tXOnFwUlXfxJCQkKDc3V6WlpRHbS0tLNXLkyGhMCQAAGBK1X/HMmzdP06ZN07BhwzRixAi99NJLOnHihGbOnBmtKQEAACOiFiiPPPKITp8+rZ/97GeqqalRdna23njjDQ0YMOBLuX2/369nn322w6+OugvWF/u6+xpZX+zr7mtkfdEVtb+DAgAAcDl8Fg8AADCHQAEAAOYQKAAAwBwCBQAAmHNDBsoLL7ygzMxM9ezZU7m5uXr77bejPaXPpbi4WHfeeaeSk5PVt29fPfTQQzpy5EjEmMcee0w+ny/ictddd0WMaW5u1ty5c5WWlqakpCRNnDhRJ0+e/DKXcklLly7tMPdgMOjtd85p6dKlCoVCSkxM1OjRo1VZWRlxHVbXdtHAgQM7rNHn82n27NmSYu/4vfXWW3rggQcUCoXk8/n02muvRezvqmNWX1+vadOmKRAIKBAIaNq0aTpz5sx1Xt2V19fa2qqnn35aOTk5SkpKUigU0g9/+EN99NFHEdcxevToDsf00UcfNbE+6erHsKvukxaPoaRLPh59Pp9++ctfemMsH8PP87wQq4/DGy5QXn31VRUWFmrJkiU6cOCA7rnnHuXn5+vEiRPRntpVlZWVafbs2dq3b59KS0v16aefKi8vT+fOnYsYd//996umpsa7vPHGGxH7CwsLtWXLFpWUlGj37t06e/asCgoK1NbW9mUu55Juv/32iLkfOnTI27d8+XKtWLFCq1evVnl5uYLBoMaPH6/GxkZvjOW1SVJ5eXnE+i7+scKHH37YGxNLx+/cuXMaMmSIVq9efcn9XXXMpkyZooMHD2rbtm3atm2bDh48qGnTpkV1fefPn9f+/fv1zDPPaP/+/dq8ebP++c9/auLEiR3GPv744xHH9MUXX4zYH631SVc/hlLX3CctHkNJEeuqqanRyy+/LJ/Pp+9///sR46wew8/zvBCzj0N3g/nOd77jZs6cGbHttttuc4sWLYrSjK5dXV2dk+TKysq8bdOnT3cPPvjgZb/nzJkzLj4+3pWUlHjb/vWvf7mbbrrJbdu27XpO96qeffZZN2TIkEvuu3DhggsGg27ZsmXetk8++cQFAgH329/+1jlne22X89RTT7lbb73VXbhwwTkX28dPktuyZYv3dVcds/fff99Jcvv27fPG7N2710ly//jHP67zqv6/9uu7lHfeecdJcsePH/e2jRo1yj311FOX/R4r63Pu0mvsivuklTV+nmP44IMPujFjxkRsi6Vj2P55IZYfhzfUGZSWlhZVVFQoLy8vYnteXp727NkTpVldu3A4LElKTU2N2L5r1y717dtXgwYN0uOPP666ujpvX0VFhVpbWyN+BqFQSNnZ2SZ+BkePHlUoFFJmZqYeffRRffjhh5Kkqqoq1dbWRszb7/dr1KhR3rytr629lpYWbdy4UT/60Y8iPvQylo/fZ3XVMdu7d68CgYCGDx/ujbnrrrsUCATMrTkcDsvn83X4rLA//OEPSktL0+23364FCxZE/J9rLKzvi94nY2GNknTq1Clt3bpVM2bM6LAvVo5h++eFWH4cRvXTjL9sH3/8sdra2jp8IGF6enqHDy60zjmnefPm6e6771Z2dra3PT8/Xw8//LAGDBigqqoqPfPMMxozZowqKirk9/tVW1urhIQE9e7dO+L6LPwMhg8frt///vcaNGiQTp06pZ///OcaOXKkKisrvbld6tgdP35ckkyv7VJee+01nTlzRo899pi3LZaPX3tddcxqa2vVt2/fDtfft29fU2v+5JNPtGjRIk2ZMiXig9emTp2qzMxMBYNBHT58WIsXL9bf/vY379d71tfXFfdJ62u8aMOGDUpOTtakSZMitsfKMbzU80IsPw5vqEC56LP/tyr996C232bdnDlz9N5772n37t0R2x955BHv39nZ2Ro2bJgGDBigrVu3dnjQfZaFn0F+fr7375ycHI0YMUK33nqrNmzY4L0o71qOnYW1XcratWuVn58f8bHjsXz8Lqcrjtmlxltac2trqx599FFduHBBL7zwQsS+xx9/3Pt3dna2srKyNGzYMO3fv19Dhw6VZHt9XXWftLzGi15++WVNnTpVPXv2jNgeK8fwcs8LUmw+Dm+oX/GkpaWpR48eHWqvrq6uQ11aNnfuXL3++uvauXOn+vfvf8Wx/fr104ABA3T06FFJUjAYVEtLi+rr6yPGWfwZJCUlKScnR0ePHvXezXOlYxdLazt+/Lh27NihH//4x1ccF8vHr6uOWTAY1KlTpzpc/7///W8Ta25tbdXkyZNVVVWl0tLSq35s/dChQxUfHx9xTC2vr71ruU/GwhrffvttHTly5KqPScnmMbzc80IsPw5vqEBJSEhQbm6ud1ruotLSUo0cOTJKs/r8nHOaM2eONm/erDfffFOZmZlX/Z7Tp0+rurpa/fr1kyTl5uYqPj4+4mdQU1Ojw4cPm/sZNDc36+9//7v69evnnV797LxbWlpUVlbmzTuW1rZu3Tr17dtXEyZMuOK4WD5+XXXMRowYoXA4rHfeeccb89e//lXhcDjqa74YJ0ePHtWOHTvUp0+fq35PZWWlWltbvWNqeX2Xci33yVhY49q1a5Wbm6shQ4ZcdaylY3i154WYfhxel5feGlZSUuLi4+Pd2rVr3fvvv+8KCwtdUlKSO3bsWLSndlVPPvmkCwQCbteuXa6mpsa7nD9/3jnnXGNjo5s/f77bs2ePq6qqcjt37nQjRoxwX/va11xDQ4N3PTNnznT9+/d3O3bscPv373djxoxxQ4YMcZ9++mm0luacc27+/Plu165d7sMPP3T79u1zBQUFLjk52Ts2y5Ytc4FAwG3evNkdOnTI/eAHP3D9+vWLibV9Vltbm7vlllvc008/HbE9Fo9fY2OjO3DggDtw4ICT5FasWOEOHDjgvYulq47Z/fff7wYPHuz27t3r9u7d63JyclxBQUFU19fa2uomTpzo+vfv7w4ePBjxmGxubnbOOffBBx+45557zpWXl7uqqiq3detWd9ttt7k77rjDxPqutsauvE9aPIYXhcNh16tXL7dmzZoO32/9GF7tecG52H0c3nCB4pxzv/nNb9yAAQNcQkKCGzp0aMTbdC2TdMnLunXrnHPOnT9/3uXl5bmvfvWrLj4+3t1yyy1u+vTp7sSJExHX09TU5ObMmeNSU1NdYmKiKygo6DAmGh555BHXr18/Fx8f70KhkJs0aZKrrKz09l+4cME9++yzLhgMOr/f7+6991536NChiOuwurbP+stf/uIkuSNHjkRsj8Xjt3PnzkveJ6dPn+6c67pjdvr0aTd16lSXnJzskpOT3dSpU119fX1U11dVVXXZx+TOnTudc86dOHHC3XvvvS41NdUlJCS4W2+91f3kJz9xp0+fNrG+q62xK++TFo/hRS+++KJLTEx0Z86c6fD91o/h1Z4XnIvdx6Hv/y0QAADAjBvqNSgAACA2ECgAAMAcAgUAAJhDoAAAAHMIFAAAYA6BAgAAzCFQAACAOQQKAAAwh0ABAADmECgAAMAcAgUAAJhDoAAAAHP+D7Zwc6i8XOjZAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "token_counts = []\n", + "max_length = 0\n", + "for row in df.iterrows():\n", + " l = len(tokenizer.encode(row[1]['input']))\n", + " token_counts.append(l)\n", + " if l > max_length:\n", + " max_length = l+10\n", + "\n", + "# plot the token counts\n", + "import matplotlib.pyplot as plt\n", + "plt.hist(token_counts, bins=15)\n", + "\n", + "modu = max_length % 256\n", + "max_length = max_length + 256 - modu" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error: no hardware_name found\n", + "{'bi-metal thermostat': 'hardware_is_owned', 'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'PSU': 'hardware_is_owned', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'fan': 'hardware_is_owned', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'server power supplies': 'hardware_is_owned', 'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'IBM 2 or 2.8kw PSU': 'hardware_is_owned', 'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'GekkoScience breakouts': 'hardware_is_owned', 'hardware_is_owned': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': True}\n", + "error\n", + "error\n", + "error: no hardware_name found\n", + "{'S17 pro': {'hardware_name': 'S17 pro', 'hardware_is_owned': False}}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error\n", + "error\n", + "error: no hardware_name found\n", + "{'S17': {'hardware_name': 'S17', 'hardware_is_owned': True}}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_name found\n", + "{'HP1200w platinum power supplies': 'hardware_is_owned', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'hardware_hashboards': 'hashboards', 'hardware_is_owned': True}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_name found\n", + "{'APW5 PSU': 'Bitmain', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'AUC serial adapter': 'hardware_is_owned', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'power_supply': 'APW3++', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': True}\n", + "error: no hardware_name found\n", + "{'T9+': {'hardware_name': 'T9+', 'hardware_is_owned': True}}\n", + "error: no hardware_name found\n", + "{'R4': {'hardware_name': 'R4', 'hardware_is_owned': False}}\n", + "error: no hardware_name found\n", + "{'APW 1600 power supply': {'hardware_name': 'APW 1600 power supply', 'hardware_is_owned': True}}\n", + "error: no hardware_name found\n", + "{'2300W PSU': {'hardware_name': '2300W PSU', 'hardware_is_owned': True}}\n", + "error: no hardware_name found\n", + "{'S5': 'S5', 'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'mining equipment': '$1.12 million worth', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'s17 pro': '16x', 'hardware_is_owned': True}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'hardware_is_owned': False}\n", + "error: no hardware_name found\n", + "{'Antminer s9': 'hardware_is_owned', 'hardware_is_owned': True}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': False}\n", + "error: no hardware_is_owned found\n", + "{'hardware_name': True}\n", + "error\n", + "hardware_discussions\n", + "error: no hardware_name found\n", + "{'top card': 'top slot card', 'hardware_is_owned': True}\n", + "error: no hardware_name found\n", + "{'mSD': {'hardware_name': 'mSD card', 'hardware_is_owned': True}}\n" + ] + } + ], + "source": [ + "\n", + "\n", + "# Function to extract hardware mentions and their ownership status\n", + "def extract_hardware_info(row):\n", + " hardware_info = json.loads(row['output'])\n", + " # hardware_dict = {item['hardware_name'].lower(): item['hardware_is_owned'] for item in hardware_info}\n", + " hardware_dict = {}\n", + " for item in hardware_info:\n", + " # if item is a string there is a problem\n", + " if isinstance(item, str):\n", + " print(\"error\")\n", + " print(item)\n", + " continue\n", + " if 'hardware_name' in item:\n", + " hardware_name = item['hardware_name']\n", + " else:\n", + " print(\"error: no hardware_name found\")\n", + " print(item)\n", + " continue\n", + " if 'hardware_is_owned' in item:\n", + " hardware_is_owned = item['hardware_is_owned']\n", + " else:\n", + " print(\"error: no hardware_is_owned found\")\n", + " print(item)\n", + " continue\n", + " hardware_dict[str(hardware_name)] = hardware_is_owned\n", + " return hardware_dict\n", + "\n", + "# Apply the function to each row\n", + "df['hardware_info'] = df.apply(extract_hardware_info, axis=1)\n", + "val['hardware_info'] = val.apply(extract_hardware_info, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Function to encode texts and labels\n", + "def encode_texts_and_labels(row):\n", + " input_text = row['input']\n", + " hardware_info = row['hardware_info']\n", + " encoding = tokenizer(input_text, truncation=True, padding='max_length', max_length=max_length, return_tensors=\"pt\")\n", + " labels = [0] * encoding.input_ids.size(1) # Initialize labels as 0 (not hardware or not owned)\n", + " \n", + " # Update labels for hardware mentions\n", + " for word, owned in hardware_info.items():\n", + " # if owned:\n", + " # print(word)\n", + " word_ids = tokenizer.encode(word, add_special_tokens=False)\n", + " for i in range(len(encoding.input_ids[0]) - len(word_ids) + 1):\n", + " if encoding.input_ids[0][i:i+len(word_ids)].tolist() == word_ids:\n", + " labels[i:i+len(word_ids)] = [2 if owned else 1] * len(word_ids)\n", + " \n", + " # for l in labels:\n", + " # if l == 1:\n", + " # print(\"yes\")\n", + " # break\n", + " encoding['labels'] = torch.tensor([labels])\n", + " return encoding\n", + "\n", + "# Apply the function to each row and collect encoded texts and labels\n", + "encoded_texts_and_labels = [encode_texts_and_labels(row) for _, row in df.iterrows()]\n", + "encoded_texts_and_labels_val = [encode_texts_and_labels(row) for _, row in val.iterrows()]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "torch.Size([1, 2048])\n", + "torch.Size([1, 2048])\n", + "torch.Size([1, 2048])\n", + "2860\n" + ] + } + ], + "source": [ + "print(encoded_texts_and_labels[0]['input_ids'].shape)\n", + "print(encoded_texts_and_labels[0]['labels'].shape)\n", + "print(encoded_texts_and_labels[0]['attention_mask'].shape)\n", + "print(len(encoded_texts_and_labels))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, HTML\n", + "import pandas as pd\n", + "import numpy as np\n", + "from torch import save, load" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_colored_html(data):\n", + " \"\"\"Generate HTML string for colored text based on provided data.\"\"\"\n", + " \n", + " def get_color(value):\n", + " \"\"\"Compute color for a given value\"\"\"\n", + " # Convert the value to a value between 0 (for 0.4) and 1 (for 0.6)\n", + " value = float(value)\n", + " # value = np.clip(value, 0.4, 0.6)\n", + " # normalized_value = (value - 0.4) / 0.2\n", + "\n", + " normalized_value = float(value)\n", + " \n", + " # Linearly interpolate between red (0) and green (1)\n", + " red = 255 * (1 - normalized_value)\n", + " green = 255 * normalized_value\n", + " \n", + " return f'rgb({int(red)}, {int(green)}, 0)'\n", + "\n", + " html_parts = []\n", + "\n", + " for text, value in data:\n", + " color = get_color(value)\n", + " html_parts.append(f'{text}')\n", + "\n", + " return ''.join(html_parts)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "▁Date:▁2013-03▁Topic:▁Remotely▁uploading▁and▁executing▁Bash▁scripts▁(BAMT)▁###▁Original▁post:▁Need▁some▁help▁uploading▁a▁bash▁file▁from▁my▁Windows▁computer▁to▁my▁BAMT▁mining▁rigs.▁I▁really▁don't▁know▁how▁to▁use▁the▁file▁transfer▁thing▁on▁ssh▁so▁I've▁been▁using▁wget▁and▁dropbox▁as▁a▁go▁between.▁This▁works▁fine▁for▁uploading▁bamt.conf▁files.▁But▁when▁I▁upload▁bash▁files▁I▁get▁this▁message▁on▁execution▁\"-bash:▁/bin/updatecg:▁/bin/bash^M:▁bad▁interpreter:▁No▁such▁file▁or▁directory\"▁This▁happens▁I▁believe▁because▁when▁I▁try▁to▁edit▁it▁with▁nano▁the▁file▁says▁converted▁from▁dos▁format.▁I▁am▁using▁notepad++▁on▁windows▁and▁saving▁it▁with▁the▁unix▁batch▁file▁option▁but▁still▁it▁must▁be▁formatted▁incorrectly.▁Can▁anyone▁help.▁All▁the▁commands▁appear▁to▁me▁perfect▁when▁viewing▁in▁nano.▁I▁can▁copy▁and▁paste▁the▁commands▁directly▁in▁a▁new▁file▁and▁the▁script▁works▁fine.▁###▁Reply▁1:▁Don't▁know▁anything▁about▁BAMT,▁butIt▁seems▁like▁you▁have▁a▁carrage▁return▁in▁there.▁So▁it's▁trying▁to▁run▁a▁shell▁called▁<truncated>▁Linux▁files▁are▁line▁feed▁terminated.▁Windows▁uses▁carriage-return▁line-feed.▁Make▁sure▁you▁remove▁all▁carriage▁returns▁before▁you▁execute▁the▁file.▁###▁Reply▁2:▁To▁transfer▁a▁file▁via▁SSH▁use▁scp.<link>▁you▁want▁to▁transfer▁file▁\"test1.txt\"▁in▁home▁folder▁of▁user1▁at▁computer1(you▁can▁put▁IP▁address▁here)▁to▁current▁folder▁type:Code:$▁scp▁<truncated>▁.Notice▁the▁dot▁at▁the▁end.▁It▁means▁you▁want▁to▁copy▁the▁file▁in▁the▁same▁folder▁you▁are▁actually▁doing▁the▁command.▁You▁can▁also▁use▁a▁path▁like▁<truncated>▁instead.▁###▁Reply▁3:▁Well▁I▁found▁out▁the▁problem.▁I▁was▁using▁notepad++▁which▁although▁doesn't▁do▁the▁spaces▁like▁a▁default▁windows▁notepad▁does▁still▁does▁the▁line▁return▁windows▁way▁(with▁default▁settings).▁Basically▁it▁worked▁for▁bamt.conf▁and▁pools▁file▁but▁no▁other▁files.▁I▁guess▁bamt▁must���ignore▁the▁carrier▁returns.▁So▁if▁you▁use▁notepad▁plus▁you▁can▁\"Edit:▁EOL▁Conversion:▁Unix▁format\"▁and▁resave▁and▁it▁worked" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2017-12▁Topic:▁Asic▁Errors▁on▁Antminers▁-▁Reboot▁Solution▁###▁Original▁post:▁Hi▁yall,I▁am▁new▁to▁Asic▁mining;▁hence,▁I▁have▁a▁question▁regarding▁asic▁errors▁on▁my▁antminers.▁I▁saw▁some▁errors▁on▁the▁asic▁status▁i.e.▁a▁mix▁of▁ooooxxooo.▁I▁rebooted▁the▁antminers▁and▁the▁errors▁go▁away.▁Is▁this▁normal?▁How▁often▁do▁you▁need▁to▁reboot▁the▁antminers?▁These▁units▁are▁still▁under▁warranty." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2014-01▁Topic:▁AsicMiner▁Cube▁38Gh/s▁1.1BTC▁Clearance▁Deal▁-▁WorldWide▁###▁Original▁post:▁Hi▁FolksHaving▁a▁clearance▁sale▁of▁the▁Asicminer▁cubes▁direct▁from▁factory▁in▁chinaPrice▁is▁1.1BTC▁per▁unit▁inc▁delivery▁by▁DHLMore▁than▁1▁unit▁price▁1.05BTCMore▁than▁2▁unit▁price▁1BTCPlease▁ask▁any▁questions▁Delivery▁takes▁3▁-▁5▁daysMy▁Current▁Stock:▁###▁Reply▁1:▁How▁can▁you▁be▁clearance-ing▁them▁out▁at▁1.1▁BTC▁shipped▁when▁people▁like▁Canary▁are▁selling▁them▁at▁.75▁+▁shipping▁(which▁still▁puts▁it▁below▁.8▁each)▁###▁Reply▁2:▁He▁is▁USA▁seller,▁I▁am▁worldwide,▁Thanks▁###▁Reply▁3:▁Hi▁Crover,▁I▁noticed▁the▁dumbbell▁in▁the▁background,▁how▁much▁are▁you▁curling▁###▁Reply▁4:▁These▁can▁make▁for▁good▁dumbbells▁when▁they▁die!▁Each▁is▁10▁lb?▁###▁Reply▁5:▁Hi▁Excuse▁my▁ignorance▁but▁how▁and▁why▁would▁this▁be▁better▁then▁just▁buying▁the▁GHS▁from▁something▁like▁ceX.io▁?" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2011-12▁Topic:▁Need▁help,▁messed▁X▁up...▁###▁Original▁post:▁Have▁remote▁rig▁(ssh▁in▁from▁Mac)▁with▁Linux▁11.10▁and▁get▁<truncated>▁export▁<truncated>▁xhost▁+No▁protocol▁specifiedxhost:▁unable▁to▁open▁display▁\":0\"miner@miner2:~$▁sudo▁aticonfig▁--odgcNo▁protocol▁specifiedERROR▁-▁X▁needs▁to▁be▁running▁to▁perform▁ATI▁Overdrive(TM)▁commandsI've▁run▁aticonfig▁--initial,▁removed▁and▁reinstalled▁fglrx▁and▁xserver-xorg▁and▁tried▁reconfiguring▁xorg▁but▁it▁all▁ends▁up▁the▁same.▁Any▁suggestions?▁(I▁won't▁have▁physical▁access▁to▁the▁rig▁for▁over▁a▁week)Here▁is▁<truncated>▁cat▁<truncated>▁\"ServerLayout\"▁Identifier▁\"aticonfig▁Layout\"▁Screen▁0▁<truncated>▁0▁0▁Screen▁<truncated>▁RightOf▁<truncated>▁Screen▁<truncated>▁RightOf▁<truncated>▁Screen▁<truncated>▁RightOf▁<truncated>▁<truncated>▁\"Monitor\"▁Identifier▁<truncated>▁Option▁\"VendorName\"▁\"ATI▁Proprietary▁Driver\"▁Option▁\"ModelName\"▁\"Generic▁Autodetecting▁Monitor\"▁Option▁\"DPMS\"▁<truncated>▁\"Monitor\"▁Identifier▁<truncated>▁Option▁\"VendorName\"▁\"ATI▁Proprietary▁Driver\"▁Option▁\"ModelName\"▁\"Generic▁Autodetecting▁Monitor\"▁Option▁\"DPMS\"▁<truncated>▁\"Monitor\"▁Identifier▁<truncated>▁Option▁\"VendorName\"▁\"ATI▁Proprietary▁Driver\"▁Option▁\"ModelName\"▁\"Generic▁Autodetecting▁Monitor\"▁Option▁\"DPMS\"▁<truncated>▁\"Monitor\"▁Iden<rest▁of▁post▁truncated>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2012-06▁Topic:▁How▁to▁force▁Catalyst▁11.11▁drivers▁(and▁SDK▁2.5)▁to▁install▁on▁Xubuntu▁12.4▁###▁Original▁post:▁I▁thought▁I▁would▁share▁this▁technique▁I▁discovered▁through▁trail▁error▁for▁forcing▁the▁Catalyst▁11.11▁driver▁to▁install▁on▁Xubuntu▁12.4First▁you▁will▁need▁to▁download▁the▁drivers▁from��the▁AMD▁archive,▁you▁can▁try▁to▁wget▁it,▁but▁for▁some▁reason▁I▁have▁problems▁with▁that▁so▁I▁just▁download▁through▁a▁browser.<link>▁+x▁the▁download▁to▁make▁it▁executable▁(chmod▁+x▁<truncated>▁super▁user▁mode▁by▁using▁sudo▁before▁each▁of▁the▁following▁lines......or▁make▁life▁easy▁and▁just▁use▁sudo▁-i▁to▁get▁a▁root▁prompt.(sudo)▁sh▁<truncated>▁--buildpkg▁Ubuntu/preciseAfter▁a▁couple▁of▁dialogs▁it▁will▁finish▁and▁you▁should▁have▁three▁new▁packages▁in▁the▁<truncated>▁It▁is▁vital▁to▁run▁this▁next▁command▁and▁let▁it▁error▁<truncated>▁dpkg▁-i▁fglrx*.debThis▁step▁is▁needed▁or▁else▁the▁software▁installer▁in▁the▁next▁step▁will▁just▁download▁and▁install▁the▁latest▁drivers▁and▁SDK▁instead▁of▁repairing▁the▁dependency▁cache▁issue.Navaigate▁with▁the▁file▁manager▁to▁<truncated>▁click▁and▁open▁the▁file▁with▁the▁\"Ubuntu▁software▁manager\".The▁software▁manager▁will▁indicate▁the▁install▁has▁a▁dependency▁cache▁error,▁click▁the▁repair▁button▁to▁have▁the▁manager▁repair▁the▁cache.▁Walk▁away▁have▁a▁soda,▁beer▁whatever..it▁will<rest▁of▁post▁truncated>▁###▁Reply▁1:▁Mmmm▁beer...Btw,▁what▁does▁<truncated>▁do?▁Why▁do▁we▁have▁to▁use▁it▁for▁mining?▁###▁Reply▁2:▁It▁is▁a▁workaround▁for▁the▁100%▁CPU▁bug▁in▁ATI▁drivers.If▁your▁rig▁is▁not▁effected▁by▁that▁bug,▁the▁workaround▁will▁not▁hinder▁anything,▁so▁one▁is▁better▁off▁just▁using▁it▁all▁the▁time▁just▁in▁case.▁###▁Reply▁3:▁Great▁article!I've▁also▁described▁the▁steps▁to▁install▁catalyst▁11▁on▁xubuntu▁12.04.Take▁a▁look!Complete▁Guide▁To▁Mine▁Bitcoin▁on▁Xubuntu▁12.04<link>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2018-01▁Topic:▁Maximum▁Hot▁Operating▁Temperatures▁###▁Original▁post:▁Hi▁Folks,I▁am▁wondering▁what▁is▁the▁highest▁operating▁temperatures▁the▁S9▁antminers▁can▁withstand▁during▁the▁summer▁time?▁Thanks.▁###▁Reply▁1:▁I▁had▁hashing▁issues▁over▁the▁summer▁when▁intake▁temps▁broke▁105f▁###▁Reply▁2:▁PCB▁temps▁in▁the▁max.▁70▁c▁area▁are▁good.PCB▁temps▁over▁80▁c,▁are▁not▁so▁good▁and▁the▁miner▁might▁overheat▁in▁85-95▁c▁area.▁###▁Reply▁3:▁The▁s9▁is▁good▁to▁a▁minimum▁of▁105c▁on▁chip▁temperatures.▁Earlier▁models▁would▁not▁shut▁down▁due▁to▁heat▁until▁the▁chips▁reached▁115cThe▁s9▁will▁run▁for▁months▁at▁a▁time▁at▁100c▁chip▁temps,▁its▁not▁an▁issue.▁###▁Reply▁4:▁Yeah▁those▁are▁chips▁temps▁and▁I▁mentioned▁pcb▁temps.We▁agree▁on▁this.▁###▁Reply▁5:▁Thank▁you▁HagssFin▁and▁Fanatic.▁So▁the▁temperatures▁in▁the▁miner▁status▁is▁already▁in▁the▁Celsius?▁I▁thought▁it▁was▁in▁Fahrenheit▁and▁always▁converted▁it▁into▁Celsius.▁Fanatic▁when▁you▁say▁intake▁air▁at▁105C▁is▁it▁the▁intake▁air▁temperature▁or▁the▁temperature▁of▁the▁chips▁in▁the▁miner▁status?▁Because▁if▁its▁taking▁105C▁then▁the▁inside▁of▁the▁miner▁will▁get▁way▁too▁hot?▁How▁do▁you▁cool▁your▁farm▁in▁summer?▁I▁remember▁you▁mentioned▁you▁have▁more▁than▁1K▁miners▁under▁management▁so▁that▁would▁be▁a▁lot▁to▁manage▁specially▁in▁the▁summer▁with▁the▁hot▁spots▁and▁everything.▁Thanks▁again▁both▁of▁you.▁###▁Reply▁6:▁Yes▁it's▁Celcius....and▁Fanatic▁was▁talking▁about▁chip▁temp▁###▁Reply▁7:▁Got▁it.▁Thank▁you.▁###▁Reply▁8:▁So▁what▁happens▁in▁a▁scenarios▁when▁the▁miner▁reaches▁say▁120C?▁Do▁you▁think▁it▁will▁stop▁hashing▁or▁will▁the▁hash▁board▁die▁or▁burn?▁Thanks.▁###▁Reply▁9:▁Thermal▁shut▁down▁I▁believe.▁###▁Reply▁10:▁Hi▁Fanatic,How▁do▁you▁cool▁your▁facilities▁in▁the▁summer▁time?▁Do▁you▁use▁evaporative▁coolers?▁What▁are▁some▁of▁the▁best▁practices?▁Thanks.▁###▁Reply▁11:▁I▁really▁cant▁go▁into▁specifics▁but▁it▁really▁depends▁on▁your▁location.▁If▁you▁live▁in▁a▁high▁humidity▁area▁you▁will▁not▁have▁much▁luck▁with▁evaporative▁cooling.▁Its▁all▁about▁controlling▁air▁flow.▁Feeding▁miners▁fresh▁air▁that▁has▁not▁already▁been▁through▁a▁heat▁cycle▁will▁do▁wonders▁even▁on▁hotter▁days.Also▁I▁mention▁chip▁temps▁because▁that▁is▁all▁the▁current▁s9s▁report.▁There▁is▁no▁way▁to▁get▁PCB▁temps▁so▁you▁have▁to▁base▁what▁you▁do▁off▁of▁the▁data▁you▁have.▁###▁Reply▁12:▁Got▁it.▁Thanks.▁Any▁book▁or▁video▁you▁could▁suggest▁for▁cooling▁in▁the▁summer▁for▁bitcoin▁mining▁facility?▁###▁Reply▁13:▁I▁dont▁know▁of▁anything▁like▁that▁as▁all▁of▁my▁experience▁is▁first▁hand.This▁might▁help▁you▁understand▁which▁route▁to▁go:▁<link>▁###▁Reply▁14:▁Thank▁you▁so▁much▁for▁the▁guidance.▁This▁is▁very▁helpful." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2022-10▁Topic:▁Configure▁Antminer▁s19▁pro▁to▁reduce▁hashrate▁and▁power▁consumption▁dynamicly?▁###▁Original▁post:▁Hi▁Everyone,I▁want▁to▁use▁my▁extra▁solar▁power▁generation▁for▁Mining.▁I▁live▁in▁Australia▁and▁electricity▁power▁is▁expensive▁and▁the▁network▁buys▁my▁feed-in▁too▁cheap!▁My▁extra▁production▁is▁variable▁between▁0▁to▁5KW▁and▁I▁want▁to▁use▁only▁that▁extra▁portion.▁I▁have▁an▁Antiminer▁s19▁pro▁which▁uses▁3kwh▁(no▁ssh▁access).▁Its▁firmware▁version▁is▁\"Thu▁Jun▁9▁13:06:22▁CST▁2022\"▁and▁Log▁shows▁\"Linux▁version▁4.9.113▁<truncated>▁and▁\"type:▁Antminer▁S19▁Pro\".▁It▁has▁3▁boards,Question▁is,▁can▁I▁change▁the▁frequency▁or▁enable▁and▁disable▁Hash▁boards▁to▁somehow▁control▁the▁amount▁of▁power▁consumption▁quickly▁(I▁don't▁care▁about▁the▁hash▁rate▁or▁<truncated>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2021-02▁Topic:▁S17▁Pro▁no▁longer▁hashing.▁Bad▁ASICs▁?▁###▁Original▁post:▁My▁S17▁has▁stopped▁hashing.▁Neither▁the▁Red▁Fault▁light▁or▁the▁Green▁Normal,▁light▁are▁on.It▁just▁keeps▁cycling▁through▁the▁messages▁below.▁Is▁this▁repairable?▁Any▁suggestions?Here's▁the▁Kernel▁Log:Code:Booting▁Linux▁on▁physical▁CPU▁0x0Linux▁version▁<truncated>▁(lzq@armdev2)▁(gcc▁version▁4.8.3▁20140320▁(prerelease)▁(Sourcery▁CodeBench▁Lite▁2014.05-23)▁)▁#25▁SMP▁PREEMPT▁Fri▁Nov▁23▁15:30:52▁CST▁2018CPU:▁ARMv7▁Processor▁[413fc090]▁revision▁0▁(ARMv7),▁cr=18c5387dCPU:▁PIPT▁/▁VIPT▁nonaliasing▁data▁cache,▁VIPT▁aliasing▁instruction▁cacheMachine▁model:▁Xilinx▁Zynqcma:▁Reserved▁16▁MiB▁at▁0x0e000000Memory▁policy:▁Data▁cache▁writeallocOn▁node▁0▁totalpages:▁<truncated>▁node▁0,▁pgdat▁c0b39280,▁node_mem_map▁cde10000▁Normal▁zone:▁480▁pages▁used▁for▁memmap▁Normal▁zone:▁0▁pages▁reserved▁Normal▁zone:▁61440▁pages,▁LIFO▁batch:15percpu:▁Embedded▁12▁pages/cpu▁@cddf1000▁s19776▁r8192▁d21184▁u49152pcpu-alloc:▁s19776▁r8192▁d21184▁u49152▁<truncated>▁[0]▁0▁[0]▁1▁Built▁1▁zonelists▁in▁Zone▁order,▁mobility▁grouping▁on.▁Total▁pages:▁60960Kernel▁command▁line:▁mem=240M▁<truncated>▁<truncated>▁root=/dev/ram▁rw▁earlyprintkPID▁hash▁table▁entries:▁1024▁(order:▁0,▁4096▁bytes)Dentry▁cache▁hash▁table▁entries:▁32768▁(order:<rest▁of▁post▁truncated>▁###▁Reply▁1:▁i▁am▁looking▁at▁it.unplug▁it▁let▁it▁sit.lets▁play▁20▁questions.did▁you▁ever▁use▁any▁firmware▁other▁then▁bitmain.what▁setting▁is▁it▁on▁low▁normal▁turbo.is▁it▁the▁s17pro▁50th▁53th▁56thdo▁not▁use▁a▁different▁firmware▁but▁if▁you▁have▁always▁had▁bitmain▁firmware▁which▁one▁are▁you▁using?do▁not▁upgrade▁the▁bitmain▁firmware▁just▁tell▁which▁one▁you▁have.▁###▁Reply▁2:▁yeah▁i▁think▁he▁killed▁psu.if▁he▁killed▁the▁psu▁it▁will▁be▁hard▁to▁get▁one.as▁bitmain▁stopped▁selling▁them.▁or▁any▁other▁gear.Op▁are▁you▁usa▁based?i▁may▁have▁a▁source▁for▁psu.first▁listen▁to▁mikeyset▁to▁lowcheck▁all▁power▁plugs▁to▁psumake▁sure▁you▁have▁210-240▁power.▁###▁Reply▁3:▁No.Turbo.I▁dont▁recall.▁How▁can▁I▁find▁out?Kernel▁Version:▁Linux▁<truncated>▁#25▁SMP▁PREEMPT▁Fri▁Nov▁23▁15:30:52▁CST▁2018Hardware▁Version:▁17.10.1.3▁###▁Reply▁4:▁This▁>Code:2020-08-03▁00:03:07▁<truncated>▁Chain[0]:▁find▁0▁asic,▁times▁2Code:2020-08-03▁00:03:37▁<truncated>▁Chain[1]:▁find▁0▁asic,▁times▁2Code:2020-08-03▁00:03:48▁<truncated>▁Chain[2]:▁find▁0▁asic,▁times▁0In▁plain▁English▁implies▁that▁the▁contorl▁board▁isn't▁sensing/seeing▁any▁asics▁on▁all▁3▁hash▁boards,▁now▁if▁it▁was▁only▁on▁a▁single▁chain▁I▁would▁say▁it's▁a▁taosted▁chain,▁but▁what▁are▁the▁chances▁of▁3▁hash▁boards▁going▁down▁at▁the▁same▁time?▁I▁would▁say▁slim▁to▁nothing▁which▁leads▁me▁to▁think▁that▁this▁has▁to▁be▁a▁POWER▁related▁issue.-▁Your▁feeding▁the▁PSU▁with▁a▁higher/lower▁AC▁voltage▁that▁it▁can▁handle,▁so▁you▁need▁to▁tell▁us▁what▁is▁the▁voltageor-▁Your▁PSU▁has▁gone▁bad▁and▁needs▁replacement,▁but▁before▁you▁do▁that,▁give▁it▁a▁good▁dusting▁and▁set▁the▁miner▁on▁\"low\"▁settings,▁the▁results▁that▁come▁out▁of▁this▁troubleshooting▁will▁probably▁lead▁us▁to▁know▁what▁the▁problem▁actually▁is.▁###▁Reply▁5:▁Heres▁the▁log▁after▁switching▁to▁Low▁Power▁mode.▁Also,▁maybe▁it▁was▁there▁before,▁but▁I▁didnt▁notice▁it,▁but▁this▁time▁the▁Miner▁Status▁was▁temporarily▁displayed.▁See▁the▁screenshots▁at▁the▁<truncated>▁hash▁table▁entries:▁1024▁(order:▁0,▁4096▁bytes)CPU:▁Testing▁write▁buffer▁coherency:▁okCPU0:▁thread▁-1,▁cpu▁0,▁socket▁0,▁mpidr▁80000000Setting▁up▁static▁identity▁map▁for▁0x100000▁-▁0x100058CPU1:▁failed▁to▁boot:▁-1Brought▁up▁1▁CPUsSMP:▁Total▁of▁1▁processors▁activated▁(666.66▁BogoMIPS).CPU:▁All▁CPU(s)▁started▁in▁SVC▁mode.devtmpfs:▁initializedVFP▁support▁v0.3:▁implementor▁41▁architecture▁3▁part▁30▁variant▁9▁rev▁4clocksource:▁jiffies:▁mask:▁0xffffffff▁max_cycles:▁0xffffffff,▁max_idle_ns:▁19112604462750000▁nspinctrl▁core:▁initialized▁pinctrl▁subsystemNET:▁Registered▁protocol▁family▁16DMA:▁preallocated▁256▁KiB▁pool▁for▁atomic▁coherent▁allocationscpuidle:▁using▁governor▁menuhw-breakpoint:▁found▁5▁(+1▁reserved)▁breakpoint▁and▁1▁watchpoint▁<truncated>▁maximum▁watchpoint▁size▁is▁4▁bytes.zynq-ocm▁f800c000.ps7-ocmc:▁ZYNQ▁OCM▁pool:▁256▁KiB▁@▁0xcf880000vgaarb:▁loadedSCSI▁subsystem▁initializedusbcore:▁registered▁new▁interface▁driver▁usbfsusbcore:▁registered▁new▁interface▁driver▁hubusbcore:▁registere<rest▁of▁post▁truncated>▁###▁Reply▁6:▁@philipma1957▁Was▁it▁you▁that▁said▁somewhere▁you▁have▁a▁repair▁contact▁in▁NC▁somewhere?▁I▁got▁and▁S17+▁for▁almost▁free▁with▁my▁Bitmain▁credit▁just▁prior▁to▁the▁virus.▁One▁board▁isnt▁reading▁(temp▁sensor▁error).▁I▁created▁a▁DOA▁repair▁ticket▁with▁Bitmain,▁and▁they▁wanted▁me▁to▁ship▁it▁to▁Asia▁for▁repair▁because▁CA▁faculty▁was▁closed▁due▁to▁virus.▁The▁agreed▁to▁honor▁my▁request▁to▁wait▁for▁CA▁to▁re-open.....▁but.....▁I'm▁still▁waiting.▁It's▁running▁fine▁on▁two▁boards,▁but▁was▁wonder▁what▁kind▁of▁costs▁i▁might▁incur▁to▁ship▁to▁NC▁dude▁for▁repair.▁Might▁even▁be▁able▁to▁hand▁deliver...▁I▁have▁several▁friends▁from▁here▁in▁PA▁who▁work▁down▁there,▁and▁they▁go▁back▁n▁forth▁often.▁###▁Reply▁7:▁I▁do.▁I▁need▁to▁look▁at▁<rest▁of▁thread▁truncated>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2023-04▁Topic:▁50%▁off▁for▁antminer▁s19▁XP▁by▁bitmain▁###▁Original▁post:▁Fantastic▁deal▁but▁it▁looks▁complicated!▁<link>▁nice▁price▁for▁an▁efficient▁machine▁like▁the▁XPDiscuss!▁###▁Reply▁1:▁Actually,▁it's▁really▁hard▁for▁you▁to▁buy▁in▁the▁price▁of▁50%▁off,▁caz▁bots▁work▁now▁instead▁of▁human▁beings.▁###▁Reply▁2:▁Link▁redirects▁to▁login▁page,▁it▁doesn't▁allow▁me▁to▁read▁the▁thing,▁lol.▁Expect▁nothing▁more.▁###▁Reply▁3:▁I▁am▁trying▁to▁figure▁out▁the▁whole▁process.▁I▁have▁logged▁in▁as▁I▁am▁a▁past▁customer▁and▁got▁the▁detailsThey▁really▁do▁not▁understand▁how▁to▁communicate▁in▁the▁english▁language▁and▁the▁details,▁though▁appear▁to▁be▁spelled▁out,▁do▁not▁really▁convey▁what▁the▁heck▁is▁really▁<truncated>▁what▁is▁the▁\"Every▁1000T▁can▁redeem▁1▁Pizza▁NFT\"▁-▁or▁other▁detail▁as▁described▁in▁step▁3I▁have▁emailed▁for▁clarification▁with▁no▁response.▁I▁wish▁I▁knew▁a▁better▁way▁to▁paste▁the▁page,▁but▁here▁are▁the▁detailed▁instructions▁from▁Bitmain▁(when▁a▁customer▁signs▁in▁to▁purchase,▁which▁I▁have▁not▁yet)Step▁1Reserve▁Your▁ApplicationMarch▁22,▁10AM▁-▁April▁4,▁8AM▁(US▁Eastern▁Time)Flash▁Sale▁Deposit:▁24U/THashrate▁fee:▁23U/TRegistration▁fee:▁1U/TNumber▁of▁Reservations▁(T)▁:▁140Flash▁Sale▁Deposit:▁3220▁USDRegistration▁fee:▁140▁USDTotal▁Amount:▁3360▁USDActivity▁Rules:After▁successful▁completion▁of▁registration▁and▁payment,▁you▁need▁to▁make▁your▁purchase▁on▁April▁4▁at▁10AM(US▁Eastern▁Time)▁in▁24▁hours.If▁the▁transaction▁is▁successful,▁the▁1U/T▁registration▁fee▁will▁be▁refunded▁in▁ANTMINER▁coupons▁to▁your▁account,and▁the▁23U/T▁hashrate▁fee▁will▁be▁applied▁to▁the▁purchase▁of▁miners.If▁the▁purchase▁is▁unsuccessful,▁t<rest▁of▁post▁truncated>▁###▁Reply▁4:▁You▁need▁an▁account▁to▁read▁the▁deal.▁You▁need▁to▁sign▁in▁to▁see▁the▁deal.The▁deal▁is▁so▁fucking▁complicated▁only▁a▁moron▁would▁do▁it.As▁I▁said▁they▁still▁owe▁me▁a▁30%▁off▁coupon▁from▁a▁2000▁dollar▁purchase.That▁deal▁had▁so▁much▁fine▁print▁and▁they▁said▁fuck▁off▁to▁me.I▁have▁purchased▁over▁250000▁in▁gear▁from▁them▁and▁I▁have▁sent▁more▁than▁1,000,000▁in▁other▁purchases▁to▁them.And▁they▁fucked▁me▁the▁coupon.▁I▁will▁not▁buy▁directly▁from▁them▁unless▁they▁fix▁what▁they▁did.It▁is▁a▁principle▁thing▁for▁me.I▁recommend▁no▁one▁buying▁from▁them▁directly▁as▁they▁have▁been▁terrible▁the▁last▁two▁years.▁###▁Reply▁5:▁Even▁more▁fucked▁is▁the▁NFT▁part.Promoting▁putting▁garbage▁in▁blockchains▁is▁just▁ridiculous.▁###▁Reply▁6:▁There▁is▁an▁extensive▁debate▁on▁the▁NFT▁subject,▁people▁have▁mixed▁views▁on▁the▁subject,▁but▁to▁Bitmain,▁the▁deal▁is▁simple,▁more▁data▁on▁the▁blockchain▁=▁more▁fees▁per▁block,▁so▁they▁make▁more▁money▁by▁1-▁Keeping▁the▁block▁fees▁on▁their▁PPLNS▁Antpool.▁2-▁Selling▁their▁gears▁for▁a▁higher▁price▁since▁now▁the▁gears▁make▁more▁BTC.Ya,▁that▁stuff▁could▁be▁bad▁for▁Bitcoin▁in▁the▁long▁term,▁but▁as▁far▁as▁the▁short-term▁gains▁are▁concerned,▁Bitmain▁would▁love▁to▁flood▁the▁blockchain▁with▁all▁things.▁###▁Reply▁7:▁looks▁like▁another▁nonsense▁post▁who▁buys▁this▁equipment?▁everyone▁hates▁bitmain▁according▁to▁almost▁every▁post▁so▁why▁do▁people▁continue▁to▁purchase▁it?▁no▁way▁in▁hell▁some▁rando▁company▁makes▁the▁best▁miners.▁i▁dont▁buy▁it▁bitmain▁is▁ibm,▁intel,▁nvidia,▁amd▁or▁all▁of▁the▁above.▁its▁so▁gross▁the▁amount▁of▁shilling▁that▁goes▁on▁and▁this▁is▁the▁forum▁that▁is▁the▁leading▁force▁for▁bitcoin?▁looooooli▁guess▁crypto▁is▁a▁dummy▁check▁the▁world▁does▁not▁need▁bitcoin▁if▁this▁is▁the▁standards.▁this▁forum▁is▁trash.▁###▁Reply▁8:▁so▁happy▁for▁you▁to▁post.as▁I▁said▁I▁do▁not▁buy▁from▁bitmain▁directly▁anymore.My▁last▁four▁units▁purchased▁were▁whatsminer▁m30▁and▁m50have▁a▁nice▁day▁###▁Reply▁9:▁NFT's▁are▁an▁attack▁and▁an▁expliot.They▁dramatically▁reduce▁the▁number▁of▁transactions▁confirmed▁per▁block.They▁increase▁the▁number▁of▁unconfirmed▁transactions,▁which▁has▁been▁high▁for▁quite▁a▁while.Currently,▁there's▁about▁10▁non-NFT▁blocks▁of▁outstanding▁transactions,▁or▁46▁NFT▁blocks▁of▁outstanding▁transactions.Not▁sure▁how▁anyone▁could▁call▁that▁anything▁but▁bad▁for▁BTC.▁###▁Reply▁10:▁Well,▁we▁don't▁care▁about▁altcoins▁do▁we?▁Problem▁is▁spamming▁Bitcoin▁and▁devs▁doing▁nothing▁about▁it.The▁excuse▁that▁mining▁must▁be▁kept▁alive▁at▁\"any\"▁cost,▁can▁only▁come▁from▁people▁who▁don't▁care▁about▁Bitcoin▁and▁only▁want▁to▁make▁the▁most▁money▁out▁of▁it▁and▁leave▁when▁it▁clogs▁perhaps▁having▁some▁altcoin▁they▁secretly▁push▁ready▁to▁take▁its▁place...And▁Bitmain▁stopped▁being▁a▁Bitcoin▁company▁the▁moment▁they▁manufactured▁ASICs▁for▁other▁algorithms,▁or▁tried▁forking▁Bitcoin.▁###▁Reply▁11:▁Bitmain▁is▁$$▁per▁watt▁coin.▁No▁more▁no▁less.And▁BTC▁has▁a▁real▁issue▁long▁term.Not▁enough▁rewards▁for▁miners▁means▁miners▁will▁leave▁the▁coin▁and▁mine▁a▁better▁algo.Scrypt▁is▁better▁for▁miners▁in▁the▁long▁run.▁maybe▁Satoshi▁wanted▁a▁electrical▁power▁to▁coin▁payment▁system.He▁built▁it.▁It▁kind▁of▁works.But▁i▁would▁argue▁scrypt▁has▁more▁staying▁power▁than▁btc.I▁will▁be▁dead▁before▁that▁plays▁out.▁say▁2060▁I▁could▁be▁wrong▁but▁I▁think▁at▁103▁I▁will▁be▁gone▁and▁m8ners▁will▁favor▁scrypt▁over▁sha▁256▁###▁Reply▁12:▁Why▁<rest▁of▁thread▁truncated>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "▁Date:▁2022-08▁Topic:▁How▁would▁you▁spend▁$50k-$60k▁for▁mining?▁###▁Original▁post:▁Purely▁Interested▁in▁what▁ASICS▁people▁would▁buy▁and▁why.▁Lets▁say▁its▁based▁on▁mining▁on▁a▁3▁Phase▁supply.▁Would▁it▁be▁all▁bitmain▁s19s▁&▁L7s?▁Or▁would▁you▁mix▁it▁up▁with▁Goldshell▁for▁KDA▁and▁maybe▁take▁a▁risk▁on▁E9▁for▁ETC?▁###▁Reply▁1:▁3-phase▁supply▁is▁*mostly*▁irrelevant▁as▁all▁miners▁use▁single▁phase▁power.▁Where▁3-phase▁does▁come▁into▁play▁is▁if▁you▁use▁phase-to-neutral▁connections▁for▁breaking▁out▁the▁power,▁you▁MUST▁ensure▁the▁resulting▁voltage▁is▁between▁208-240VAC▁and▁in▁most▁countries▁your▁phase-to-neutral▁voltage▁will▁either▁be▁too▁high▁or▁too▁low.▁###▁Reply▁2:▁Well▁this▁is▁btc▁only▁and▁people▁frown▁on▁discussing▁other▁coins▁in▁this▁section.But▁I▁havebtc▁asicsltc/doge▁asicsgpusI▁constantly▁<truncated>▁attempt▁to▁balance▁risks▁that▁way.I▁make▁less▁money▁this▁way.▁ie▁going▁all▁in▁on▁the▁right▁gear▁at▁the▁right▁time▁will▁earn▁more▁money▁than▁my▁way.but▁I▁lose▁less▁when▁shit▁happens.▁ie▁going▁all▁in▁on▁the▁wrong▁gear▁at▁the▁wrong▁time▁can▁be▁crushing.▁###▁Reply▁3:▁NotFuzzyWarm▁answered▁you▁about▁that,▁and▁I▁remembered▁you▁asked▁about▁your▁electrical▁setup▁and▁you're▁fixated▁on▁these▁issues▁when▁the▁main▁question▁is▁obviously▁the▁same,▁how▁much▁per▁kWh▁and▁how▁much▁can▁you▁really▁pull▁out▁at▁that▁price?Seeing▁you're▁from▁UK,▁and▁knowing▁you▁want▁to▁mine▁from▁your▁own▁garage▁so▁it's▁probably▁residential▁rates,▁are▁you▁sure▁about▁this?▁I▁have▁(not▁sure▁for▁long)▁still▁single▁digits▁rates▁and▁I▁wouldn't▁throw▁a▁penny▁at▁gear▁now." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# to verify correctness, let's display the first sample with tokens highlighted when label is 1\n", + "id = 1\n", + "for id in range(10):\n", + " input_ids = encoded_texts_and_labels[id]['input_ids'][0]\n", + " labels = encoded_texts_and_labels[id]['labels'][0]*0.5\n", + " tokens = tokenizer.convert_ids_to_tokens(input_ids.numpy())\n", + " arr = [(token, label) for token, label in zip(tokens, labels)]\n", + " display(HTML(generate_colored_html(arr[1:])))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from torch.utils.data import Dataset, DataLoader\n", + "\n", + "class HardwareDataset(Dataset):\n", + " def __init__(self, encodings):\n", + " self.encodings = encodings\n", + "\n", + " def __getitem__(self, idx):\n", + " # return self.encodings[idx]\n", + " return {\n", + " 'input_ids': self.encodings[idx]['input_ids'].flatten(),\n", + " 'labels': self.encodings[idx]['labels'].flatten(),\n", + " 'attention_mask': self.encodings[idx]['attention_mask'].flatten()\n", + " }\n", + "\n", + " def __len__(self):\n", + " return len(self.encodings)\n", + "\n", + "# Create the dataset\n", + "train_dataset = HardwareDataset(encoded_texts_and_labels)\n", + "val_dataset = HardwareDataset(encoded_texts_and_labels_val)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of LongformerForTokenClassification were not initialized from the model checkpoint at severinsimmler/xlm-roberta-longformer-base-16384 and are newly initialized: ['classifier.bias', 'classifier.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "import torch\n", + "from transformers import TrainingArguments, Trainer\n", + "\n", + "# from longformer.longformer import Longformer, LongformerConfig\n", + "# from longformer.sliding_chunks import pad_to_window_size\n", + "\n", + "from transformers import LongformerForTokenClassification, XLMRobertaTokenizerFast\n", + "model = LongformerForTokenClassification.from_pretrained('severinsimmler/xlm-roberta-longformer-base-16384', num_labels=3)\n", + "tokenizer = XLMRobertaTokenizerFast.from_pretrained('xlm-roberta-base')\n", + "config = model.config\n", + "tokenizer.model_max_length = config.max_position_embeddings\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "model = model.to('cpu')\n", + "#only the last layer and the classifier should be trained\n", + "for param in model.parameters():\n", + " param.requires_grad = False\n", + "for param in model.classifier.parameters():\n", + " param.requires_grad = True\n", + "# for param in model.longformer.encoder.layer[-1].parameters():\n", + "# param.requires_grad = True\n", + "# for param in model.longformer.encoder.layer[-2].parameters():\n", + "# param.requires_grad = True\n", + "# for param in model.longformer.encoder.layer[-3].parameters():\n", + "# param.requires_grad = True\n", + " \n", + "for layer in model.longformer.encoder.layer[-8:]:\n", + " for param in layer.parameters():\n", + " param.requires_grad = True" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [2145/2145 21:08, Epoch 3/3]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation Loss
10.0814000.026701
20.0247000.024375
30.0218000.024706

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Checkpoint destination directory ./results/checkpoint-500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n", + "Checkpoint destination directory ./results/checkpoint-1000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n" + ] + } + ], + "source": [ + "from datasets import load_metric\n", + "\n", + "def compute_metrics(p):\n", + " metric = load_metric(\"accuracy\")\n", + " metric_f1 = load_metric(\"f1\")\n", + " accuracy = metric.compute(predictions=p.predictions.argmax(-1), references=p.label_ids)\n", + " f1 = metric_f1.compute(predictions=p.predictions.argmax(-1), references=p.label_ids, average='macro')\n", + " return {\"accuracy\": accuracy[\"accuracy\"], \"f1\": f1}\n", + "\n", + "from transformers import AdamW, get_linear_schedule_with_warmup\n", + "from torch.optim.lr_scheduler import LambdaLR\n", + "\n", + "class EpochBasedLRScheduler(LambdaLR):\n", + " def __init__(self, optimizer, lr_per_epoch, len_train_loader, last_epoch=-1):\n", + " self.lr_per_epoch = lr_per_epoch\n", + " self.len_train_loader = len_train_loader\n", + " super(EpochBasedLRScheduler, self).__init__(optimizer, self.lr_lambda, last_epoch=last_epoch)\n", + "\n", + " def lr_lambda(self, current_step: int):\n", + " current_epoch = current_step // self.len_train_loader\n", + " if current_epoch < len(self.lr_per_epoch):\n", + " return self.lr_per_epoch[current_epoch] / self.base_lrs[0]\n", + " return 1.0\n", + "\n", + " \n", + "from transformers import TrainerCallback, TrainerControl, TrainingArguments\n", + "\n", + "class PrintLearningRateCallback(TrainerCallback):\n", + " def on_epoch_end(self, args: TrainingArguments, state: TrainerControl, control: TrainerControl, **kwargs):\n", + " lr = state.log_history[-1]['learning_rate']\n", + " print(f\"Learning rate at end of epoch {state.epoch}: {lr}\")\n", + "\n", + "\n", + "\n", + "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-7)\n", + "lr_per_epoch = [\n", + " 1e-5,\n", + " # 1e-5,\n", + " # 1e-5,\n", + " # 1e-5,\n", + " # 1e-5,\n", + " # 1e-5,\n", + " 2e-6,\n", + " # 2e-6,\n", + " # 2e-6,\n", + " # 1e-6,\n", + " # 1e-6,\n", + " # 1e-6,\n", + "]\n", + "\n", + "\n", + "class EpochAverageLossCallback(TrainerCallback):\n", + " def __init__(self):\n", + " self.cumulative_loss = 0.0\n", + " self.batch_count = 0\n", + "\n", + " def on_train_begin(self, args, state, control, **kwargs):\n", + " # Reset at the beginning of training\n", + " self.cumulative_loss = 0.0\n", + " self.batch_count = 0\n", + "\n", + " def on_step_end(self, args, state, control, **kwargs):\n", + " # Accumulate losses and increment batch count after each step\n", + " self.cumulative_loss += state.log_history[-1]['loss'] if state.log_history else 0\n", + " self.batch_count += 1\n", + "\n", + " def on_epoch_end(self, args, state, control, **kwargs):\n", + " # Compute average loss and print it\n", + " average_epoch_loss = self.cumulative_loss / self.batch_count\n", + " print(f\"Average training loss over epoch {state.epoch}: {average_epoch_loss:.4f}\")\n", + "\n", + " # Reset for next epoch\n", + " self.cumulative_loss = 0.0\n", + " self.batch_count = 0\n", + "\n", + "\n", + "retrain = True\n", + "# retrain = False\n", + "\n", + "training_args = TrainingArguments(\n", + " output_dir='./results',\n", + " num_train_epochs=len(lr_per_epoch),\n", + " per_device_train_batch_size=4,\n", + " per_device_eval_batch_size=4,\n", + " warmup_steps=0,\n", + " weight_decay=0.01,\n", + " logging_steps=len(train_dataset) // 4,\n", + " #lr_scheduler_type='constant',\n", + " #learning_rate=1e-9,\n", + " report_to='none',\n", + " evaluation_strategy='epoch',\n", + ")\n", + "\n", + "len_train_loader = len(train_dataset) // training_args.per_device_train_batch_size\n", + "scheduler = EpochBasedLRScheduler(optimizer, lr_per_epoch, len_train_loader)\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + " eval_dataset=val_dataset,\n", + " # compute_metrics=compute_metrics,\n", + " optimizers=(optimizer, scheduler),\n", + " # callbacks=[PrintLearningRateCallback(), EpochAverageLossCallback()]\n", + ")\n", + "\n", + "if retrain:\n", + " model.train()\n", + " trainer.train()\n", + " torch.save(model, '/kaggle/working/model.pt' if kaggle else 'bert.pt')\n", + " model.eval()\n", + "else:\n", + " model = torch.load('/kaggle/working/model.pt' if kaggle else 'bert.pt')\n", + " model.eval()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2021-07▁Topic:▁Whats▁the▁best▁mining▁equipment▁on▁the▁market▁today?▁###▁Original▁post:▁I▁have▁a▁place▁which▁is▁pretty▁cheap▁in▁electricity▁and▁would▁have▁good▁circulation▁being▁a▁warehouse.▁The▁warehouse▁is▁huge▁and▁while▁it▁is▁used▁for▁other▁things▁there▁would▁be▁a▁dedicated▁area▁for▁the▁miners▁so▁they▁could▁run▁uninterrupted▁24/7.▁The▁only▁issue▁is▁there▁are▁so▁many▁miners▁out▁there▁today▁and▁a▁lot▁of▁them▁are▁either▁sold▁out▁or▁on▁preorder▁and▁I▁am▁wondering▁what▁would▁be▁the▁best▁investment▁for▁a▁medium▁scale▁operation?▁###▁Reply▁1:▁There▁is▁no▁\"one-size-fits-all\"▁when▁it▁comes▁to▁choosing▁a▁mining▁gear,▁it's▁a▁combination▁of▁a▁few▁factors.1-▁Your▁power▁rate▁and▁voltage2-▁Your▁budget3-▁Your▁power▁and▁space▁capcitityThese▁are▁a▁few▁major▁factors▁that▁need▁to▁be▁considered,▁there▁are▁a▁few▁more,▁but▁you▁basically▁want▁to▁evaluate▁all▁things▁to▁make▁the▁right▁decision,▁the▁only▁common▁thing▁would▁be▁to▁avoid▁mining▁gears▁that▁are▁known▁to▁have▁a▁high▁failure▁rate,▁other▁than▁that,▁it's▁case-by-case.▁###▁Reply▁2:▁It▁also▁depends▁on▁how▁many▁miners▁you're▁able▁to▁fit▁in▁the▁warehouse,▁since▁you'll▁want▁to▁maximize▁the▁space▁used▁while▁staying▁within▁budget.▁I▁don't▁believe▁miner▁dimensions▁vary▁too▁much▁but▁if▁it▁means▁buying▁second-hand▁or▁slightly▁older-gen▁miners▁so▁use▁up▁all▁the▁space,▁do▁so▁(e.g▁S17▁instead▁of▁S19,▁but▁not▁an▁S9▁for▁example).▁###▁Reply▁3:▁Used▁S17+▁is▁the▁best▁investment▁for▁me,▁good▁pricing,▁high▁hashrate▁and▁good▁cost▁efficiency..▁###▁Reply▁4:▁Out▁of▁tens▁of▁models▁that▁the▁mining▁industry▁has▁ever▁known,▁the▁S17+▁is▁in▁the▁top▁3▁worst▁mining▁gears▁ever,▁it▁has▁one▁of▁the▁highest▁failure▁rates▁among▁its▁generation▁model▁which▁is▁the▁17▁series,▁and▁the▁17▁series▁are▁by▁far▁the▁WORST▁performing▁mining▁gears,▁there▁is▁a▁good▁reason▁as▁to▁why▁they▁sell▁them▁a▁lot▁cheaper▁than▁other▁models▁which▁are▁even▁less▁effienct.▁###▁Reply▁5:▁The▁s17+The▁s17ethe▁t17ethe▁t17+.▁all▁were▁bad▁for▁me.the▁s17pro▁has▁beed▁good.▁the▁50th▁and▁the▁53th▁both▁do▁well▁for▁me.and▁the▁two▁t17▁were▁okay.at▁op▁what▁is▁cheap▁power▁2▁cents▁3▁cents▁4▁cents▁5▁cents▁a▁kwatt▁or▁something▁elseand▁how▁many▁watts▁can▁you▁do▁at▁220-240▁volts?It▁matters▁as▁the▁s9▁could▁work▁if▁you▁have▁really▁cheap▁power▁,space▁and▁lots▁of▁power.You▁can▁get▁s9▁with▁psu▁in▁large▁buys▁of▁500▁units▁cheap.that▁would▁be▁1300▁x▁500▁=▁650kwatts.it▁would▁be▁6.▁or▁7▁ph▁which▁is▁around▁2000▁usd▁worth▁of▁coin▁a▁day.if▁you▁buy▁the▁gear▁at▁150k▁usd▁and▁your▁power▁is▁2▁cents▁it▁costs.▁312▁a▁day▁so▁you▁net2000-312=▁1688▁a▁day▁in▁100▁days▁the▁gear▁is▁paid▁off.but▁you▁need▁two▁cent▁power.▁and▁650▁kwatts▁an▁hour▁to▁burn." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2018-11▁Topic:▁Best▁way▁to▁shut▁down▁an▁Avalon▁741?▁###▁Original▁post:▁For▁the▁741,▁there's▁2▁ways▁to▁shut▁it▁off▁-▁turn▁off▁the▁controller▁first▁or▁the▁main▁power▁supplies.For▁mine,▁I▁figured▁that▁if▁I▁shut▁down▁the▁controller▁first,▁that▁would▁end▁the▁data▁stream▁and▁the▁miner▁would▁go▁to▁idle.▁And▁it▁did,▁so▁I▁could▁then▁safely▁give▁it▁a▁few▁minutes▁to▁cool▁off▁before▁turning▁off▁both▁power▁supplies▁(HP▁server▁1200▁and▁750▁watt▁PSUs).▁###▁Reply▁1:▁hi,and▁how▁to▁you▁shut▁down▁the▁controller▁safely?▁###▁Reply▁2:▁That's▁what▁I▁tend▁to▁do.▁It▁just▁seems▁to▁be▁the▁gentlest▁way▁to▁power▁down▁the▁equipment.I▁actually▁run▁2▁741s▁on▁2450w▁psus▁which▁is▁why▁I▁started▁thinking▁of▁a▁way▁to▁avoid▁just▁throwing▁the▁switch.I▁tend▁to▁stop▁cgminer▁from▁the▁status▁page.▁Watch▁the▁miners▁idle▁until▁they▁are▁cooled▁down▁to▁the▁touch.▁Then▁throw▁power▁and▁disconnect.▁After▁this▁if▁I▁have▁to▁move▁the▁controller▁I▁unplug▁it." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0000001\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2019-02▁Topic:▁Announcing▁the▁WhatsMiner▁M10S▁44TH▁@▁60W/TH▁or▁55TH▁@▁65W/TH▁SHA-256▁16nm▁Miner▁###▁Original▁post:▁well▁the▁m10▁does▁34.5▁th▁at▁68▁watts▁a▁th▁which▁is▁the▁best▁gear▁I▁ever▁owned.if▁these▁do▁44th▁at▁60▁wattsand▁54th▁at▁65▁watts▁it▁would▁be▁amazing▁improvement▁.I▁did▁multiple▁videos▁of▁the▁34.5th▁m10▁on▁you▁tubeI▁now▁have▁got▁to▁get▁one▁of▁these▁###▁Reply▁1:▁That's▁a▁lot▁of▁power....yikes!▁###▁Reply▁2:▁I▁like▁the▁44Th/s▁model▁more▁because▁of▁the▁insanely▁good▁efficiency▁and▁better▁amount▁of▁load▁for▁16▁Amp▁circuit.edit:▁btw.▁see▁my▁m10▁review▁here▁<link>▁###▁Reply▁3:▁I▁posted▁3▁videos▁in▁your▁m10▁thread.these▁guys▁are▁making▁monsters.44th54thI▁guess▁I▁have▁to▁try▁to▁get▁one▁of▁them▁based▁on▁this▁unit.▁###▁Reply▁4:▁Wow,▁I▁can't▁wait▁to▁see▁the▁pricing.▁The▁55TH▁@3500▁will▁be▁almost▁maxing▁one▁of▁my▁20A/240V▁circuits▁(3840▁usable)!▁###▁Reply▁5:▁Just▁guess▁how▁I▁feel▁when▁I▁think▁about▁that▁miner▁and▁my▁16A/230V▁circuits.▁###▁Reply▁6:▁I'm▁kicking▁myself▁for▁not▁installing▁30A▁instead▁of▁20A.▁A▁larger▁subpanel▁with▁5x30A▁is▁infinitely▁better▁than▁the▁5x20A▁I▁have...▁except▁for▁the▁heat▁and▁noise.▁Lesson▁for▁people▁jumping▁into▁the▁mining▁world:▁more▁power▁is▁always▁better▁than▁less▁power!▁###▁Reply▁7:▁...▁dont▁buy.▁Sorry▁Scott,▁but▁Halong▁was▁damage▁enough▁to▁the▁<truncated>▁will▁not▁bring▁you▁money▁or▁ROI.Same▁like▁Halong▁or▁Inno.Just▁see▁what▁chip▁design▁they▁use▁(10nm▁from▁Samsung,▁14nm▁or▁16nm).But▁they▁gain▁the▁same▁efficiency▁like▁the▁odd▁and▁old▁16nm▁Bitmain▁chips.IT▁IS▁ALL▁ABOUT▁DESIGN▁and▁about▁HASH/WATT.And▁this▁chip▁design▁of▁Bitmain▁-▁translated▁to▁a▁10nm▁or▁7nm▁chip▁is▁like▁the▁one▁ring:▁To▁rule▁them▁all.7nm▁chip▁was▁announced▁8▁hours▁before▁on▁Bitmain:ALL▁BASE▁BELONG▁TO▁US▁###▁Reply▁8:▁Whatsminer▁stuff▁looks▁great.▁Easy▁yo▁be▁enthusiastic▁about▁it▁till▁pricing▁comes▁into▁play.▁Love▁what▁their▁doing▁but▁over▁priced▁and▁will▁be▁even▁less▁accessible▁till▁we▁find▁out▁how▁exemption▁turns▁out▁for▁tariffs.▁Edit.▁After▁I▁posted▁this▁I▁checked▁Pano's▁website.▁M10s▁has▁been▁removed▁and▁don't▁see▁any▁tweets▁about▁it▁anymore.▁###▁Reply▁9:▁Follows▁how▁Innosilicon▁has▁\"sold▁out\"▁of▁the▁T2T+32.▁Think▁they're▁pulling▁these▁products▁until▁they▁see▁how▁the▁tariffs▁shake▁out?▁###▁Reply▁10:▁No,▁their▁not▁concerned▁with▁tariffs▁to▁the▁US.▁I▁see▁the▁withdraw▁of▁listing▁to▁move▁more▁sales▁to▁the▁m10.▁Still▁lots▁of▁learning▁for▁marketing▁in▁this▁space.▁###▁Reply▁11:▁In▁Stock.▁2▁Day▁Shipping.▁###▁Reply▁12:▁Oh,▁weird.▁Get▁them▁while▁they're▁hot.▁I▁imagine▁pano's▁website▁is▁down.Yup,▁site▁down.▁We▁got▁pricing▁but▁not▁spec.▁Taking▁pre▁orders▁if▁buyers▁like.▁###▁Reply▁13:▁Lol,▁Philip▁you're▁a▁very▁welcome▁buyer.▁Looks▁like▁just▁a▁few▁units▁left▁in▁this▁batch.▁We're▁closing▁orders▁for▁credit▁as▁routing▁time▁is▁longer▁for▁those▁payments.55th/3500w▁was▁the▁spec▁communicated.▁###▁Reply▁14:▁maybe▁if▁you▁let▁me▁###▁Reply▁15:▁Announcing▁the▁WhatsMiner▁M10S▁44TH▁@▁60W/TH▁or▁55TH▁@▁65W/TH▁SHA-256▁16nm▁Miner:See▁the▁post▁on▁Medium:▁<link>▁the▁Tweet▁on▁Twitter:▁<link>▁many▁details▁available▁yet.▁Ships▁in▁the▁end▁of▁January▁2019.▁###▁Reply▁16:▁Is▁there▁a▁way▁to▁get▁hands▁on▁a▁demo,▁ive▁got▁40▁mw▁to▁fill▁and▁I▁need▁to▁figure▁out▁what▁hardware▁we▁want▁to▁go▁with▁sooner▁rather▁than▁later▁###▁Reply▁17:▁If▁it▁runs▁like▁the▁m10▁you▁will▁like▁it▁.My▁2▁m10's▁have▁been▁flawless.▁###▁Reply▁18:▁Yea▁I▁am▁pretty▁excited▁to▁test▁it▁out.Thanks▁Scott!▁###▁Reply▁19:▁Be▁cool▁to▁hear▁more▁about▁these.▁Our▁first▁orders▁started▁arriving▁early▁this▁week▁for▁buyers.▁###▁Reply▁20:▁i▁get▁about▁55.4TH▁(slushpool)▁@3500▁watts,▁1▁hour▁averagebut▁have▁to▁wait▁until▁24h▁average.▁seems▁promosing.i▁mean,▁16nm▁vs▁bitmain▁7nm▁BIT*H.▁and▁its▁even▁without▁asicboost▁###▁Reply▁21:▁I▁just▁received▁1▁M10S▁today,▁getting▁51▁tH/s▁and▁3213▁watts▁at▁the▁wall▁in▁low▁power▁mode.▁Exactly▁63▁W/TH.▁I▁don't▁think▁my▁PDU▁can▁handle▁regular▁power▁mode▁so▁I▁didn't▁test▁it;▁I▁was▁already▁getting▁warnings▁on▁the▁PDU▁with▁low▁power▁mode▁set.▁###▁Reply▁22:▁What▁voltage▁are▁you▁running▁at?▁208v,▁220v,▁240v?▁###▁Reply▁23:▁It▁is▁actually▁a▁3350▁Watt▁power▁supply▁unit.▁WhatsPower▁P20.▁specs-->▁<link>▁full▁load▁and▁95%▁nominal▁power▁efficiency,▁the▁theoretical▁max.▁power▁at▁wall▁is▁about▁3526▁Watt.▁###▁Reply▁24:▁Well▁that▁looks▁like▁their▁really▁playing▁fast▁and▁loose▁with▁max▁power▁draw.Is▁there▁any▁info▁regarding▁burn▁in▁for▁long▁term▁24▁hour▁operation?▁Just▁curious▁as▁I've▁always▁been▁weary▁of▁built▁un▁PSU's▁###▁Reply▁25:▁@offordscottOkay,▁thanks▁for▁the▁clarification.▁I▁got▁to▁say▁that▁I▁don't▁have▁a▁M10s.It▁seems▁that▁the▁actual▁nominal▁max.▁power▁capacity▁is▁slightly▁less▁than▁mentioned▁in▁the▁MicroBT▁website.▁###▁Reply▁26:▁HF▁do▁you▁have▁a▁microBT▁contact?▁Their▁email▁gets▁returned.▁###▁Reply▁27:▁Glad▁to▁hear▁some▁have▁made▁it▁out.▁These▁are▁the▁sold▁demo▁units▁I'm▁guessing.Can▁you▁guys▁post▁photos▁or▁screencaps▁of▁the▁logs▁and▁gui▁for▁everyone▁to▁checkout.I▁heard▁these▁were▁being▁priced▁at▁3200▁anyone▁know▁for▁sure▁if▁that's▁what▁they▁are▁going<rest▁of▁thread▁truncated>" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2014-08▁Topic:▁[Ongoing]▁Group▁buy▁for▁Antminer▁S3+▁Pre▁order▁CC▁&▁Paypal▁accepted▁$339.99▁###▁Original▁post:▁Purchase▁the▁Antminer▁S3+▁453GH/s▁MinerHosted▁by▁Maidak▁via▁Direct▁Product▁Link▁for▁group▁buy▁purchase▁$379.99▁each▁or▁$339.99▁if▁you▁buy▁4▁or▁more<link>▁More▁Information▁About▁Myself:Hello,▁My▁name▁is▁Maidak.▁Ive▁been▁a▁member▁of▁the▁forum▁since▁2011.▁I▁have▁been▁responsible▁for▁the▁management▁for▁over▁7▁years▁in▁virtual▁currency,▁bitcoin▁and▁other▁non▁refundable▁purchases▁in▁exchange▁for▁virtual▁and▁physical▁products.▁Here▁are▁links▁to▁some▁of▁my▁previously▁conducted▁group▁buys:<link>▁choose▁me?Please▁see▁here▁for▁a▁consolidated▁list▁of▁various▁forums▁and▁websites▁where▁you▁can▁view▁my▁feedback:<link>▁am▁the▁owner/operator▁of▁bitcoinminerz.com,▁a▁bitcoin▁mining▁hardware▁and▁accessory▁shop.▁This▁group▁buy▁will▁be▁conducted▁via▁the▁site▁for▁convenience.▁Check▁out▁my▁site▁here:<link>▁I▁have▁successfully▁operate▁a▁free▁escrow▁service▁in▁the▁forums:<link>▁see▁here▁for▁more▁information▁about▁my▁reputation:<link>▁I▁am▁a▁publicly▁approachable,▁genuine▁person.▁I▁have▁nothing▁to▁hide▁from▁the▁community.I▁am▁a▁TrustWho▁verified▁member:<link>▁Learn▁more▁about▁the▁TrustWho▁service▁here:<link>▁I▁aim▁to▁provide▁a▁superior▁customer▁service▁experience!" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2020-02▁Topic:▁Asic▁Anminer▁s9k▁###▁Original▁post:▁I▁tried▁a▁few▁things▁with▁my▁2nd▁hand▁S1.Simplest▁fix,▁which▁will▁get▁you▁2/3▁boards▁mining,▁is▁to▁-▁switch▁it▁all▁off-▁disconnect▁12V▁lines▁from▁the▁faulty▁board-▁check▁that▁there▁are▁no▁loose▁live▁ends▁in▁a▁bad▁place-▁power▁it▁up-▁mine▁with▁the▁remaining▁'good'▁ASIC▁boardsAntminer▁products▁appear▁to▁be▁safe▁to▁run▁with▁less▁than▁all▁boards▁powered.▁Think▁of▁all▁that▁electricity▁which▁you▁are▁saving▁by▁mining▁at▁2/3▁speed.You▁might▁also▁want▁to▁look▁at▁the▁\"overclocking\"▁method▁to▁decrease▁(rather▁than▁increase)▁clock▁rates.▁There▁can▁be▁times▁when▁decreasing▁the▁clock▁rate▁by▁about▁10%▁does▁restore▁operation▁of▁the▁bad▁board.▁###▁Reply▁1:▁We▁almost▁all▁do,▁this▁model▁is▁the▁worst▁out▁there,▁I▁have▁yet▁to▁find▁someone▁who▁has▁a▁few▁S9ks▁that▁mine▁without▁a▁problem,▁anyway▁for▁your▁case▁its▁plain▁and▁simple,▁your▁hash▁board▁number▁[1]▁\"middle▁chain\"▁has▁lost▁some▁chips▁and▁the▁miner▁refuses▁to▁mine▁as▁long▁as▁the▁hash▁board▁shows▁anything▁less▁than▁60▁Asics▁on▁each▁control▁board,▁by▁unplugging▁that▁faulty▁miner,▁the▁miner▁should▁mine▁just▁fine▁with▁the▁two▁remaining▁boards,▁alternatively▁and▁before▁you▁discount▁that▁lagging▁board,▁you▁should▁try▁the▁latest▁firmware▁version.Some▁times,▁the▁power▁supply▁cables▁or▁the▁ribbon▁cables▁that▁go▁from▁the▁control▁board▁to▁the▁hash▁board▁can▁cause▁the▁same▁issue,▁it's▁not▁very▁common▁but▁still▁worth▁to▁test,▁you▁have▁to▁swap▁the▁ribbon▁cable▁and▁the▁6▁pin▁cables▁from▁another▁board▁and▁see▁if▁you▁get▁any▁luck,▁if▁not▁then,▁well▁sadly▁you▁have▁lost▁a▁hash▁board▁which▁is▁common▁in▁all▁Bitmain▁mining▁gears▁and▁most▁common▁in▁this▁specific▁model.▁###▁Reply▁2:▁Hello▁people!I▁have▁a▁problem▁with▁my▁s9k(▁sorry▁for▁bad▁english)After▁start▁working▁second▁hash▁board▁have▁eror▁and▁dont▁work(▁added▁on▁photos)and▁have▁a▁kernel▁log▁wheare▁i▁see▁changes▁with▁voltage.I▁dont▁know▁wthat▁is▁problem▁heare(▁becouse▁i▁m▁begginer▁on▁ASIC's)Code:Booting▁Linux▁on▁physical▁CPU▁0x0Linux▁version▁<truncated>▁(lzq@armdev2)▁(gcc▁version▁4.8.3▁20140320▁(prerelease)▁(Sourcery▁CodeBench▁Lite▁2014.05-23)▁)▁#25▁SMP▁PREEMPT▁Fri▁Nov▁23▁15:30:52▁CST▁2018CPU:▁ARMv7▁Processor▁[413fc090]▁revision▁0▁(ARMv7),▁cr=18c5387dCPU:▁PIPT▁/▁VIPT▁nonaliasing▁data▁cache,▁VIPT▁aliasing▁instruction▁cacheMachine▁model:▁Xilinx▁Zynqcma:▁Reserved▁16▁MiB▁at▁0x0e000000Memory▁policy:▁Data▁cache▁writeallocOn▁node▁0▁totalpages:▁<truncated>▁node▁0,▁pgdat▁c0b39280,▁node_mem_map▁cde10000Normal▁zone:▁480▁pages▁used▁for▁memmapNormal▁zone:▁0▁pages▁reservedNormal▁zone:▁61440▁pages,▁LIFO▁batch:15percpu:▁Embedded▁12▁pages/cpu▁@cddf1000▁s19776▁r8192▁d21184▁u49152pcpu-alloc:▁s19776▁r8192▁d21184▁u49152▁<truncated>▁[0]▁0▁[0]▁1▁Built▁1▁zonelists▁in▁Zone▁order,▁mobility▁grouping▁on.▁Total▁pages:▁60960Kernel▁command▁line:▁mem=240M▁<truncated>▁<truncated>▁root=/dev/ram▁rw▁earlyprintkPID▁hash▁table▁entries:▁1024▁(o<rest▁of▁post▁truncated>▁###▁Reply▁3:▁Recently▁there▁is▁a▁post▁here▁on▁the▁forum▁that▁has▁the▁issue▁on▁the▁middle▁board▁the▁same▁as▁yours▁but▁different▁kernel▁logs.▁The▁weird▁thing▁is▁they▁also▁have▁the▁issue▁on▁the▁middle▁hashboard.▁Can▁you▁try▁to▁attach▁the▁ribbon▁and▁power▁cable▁from▁chain▁to▁chain▁[1]▁and▁then▁attach▁the▁cable▁from▁chain▁1▁to▁0?▁Let's▁see▁if▁there▁are▁changes▁in▁the▁kernel▁logs.▁If▁the▁issue▁on▁chain▁1▁is▁gone▁and▁the▁chain▁0▁shows▁some▁issue▁your▁hashboard▁is▁dead▁but▁if▁the▁error▁still▁in▁chain▁1▁there▁might▁be▁an▁issue▁from▁the▁firmware▁which▁is▁a▁common▁issue▁on▁this▁unit▁model." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2011-06▁Topic:▁Motherboard▁with▁onboard▁video▁###▁Original▁post:▁I▁do▁have▁a▁motherboard▁with▁onboard▁too▁(Gigabyte▁GA-MA78G-DS3H)▁and▁I▁can't▁use▁two▁cards.▁My▁second▁PCIe▁slot▁does▁not▁work▁with▁video▁cards.▁Either▁it▁activates▁the▁one▁in▁the▁first▁PCIe▁slot,▁or▁the▁onboard▁if▁its▁missing.▁I▁don't▁know▁about▁your▁specific▁mainboard,▁but▁I▁needed▁to▁buy▁a▁new▁one.▁###▁Reply▁1:▁Hi,I,▁considdering▁buying▁an▁ASRock▁<truncated>▁But▁it▁has▁onboard▁video.▁The▁wiki:▁<link>▁claims▁mobo's▁with▁onboard▁video▁gives▁problems.Anybody▁expierience▁with▁this▁mobo▁er▁other▁with▁onboard▁video?▁i▁it▁works▁this▁is▁an▁ideal▁mobo.Specs:<link>▁2x▁ASUS▁EAH5850▁DirectCU/2DIS/1GD5▁<link>▁AMD▁Athlon▁II▁250psu:▁Corsair▁Enthusiast▁Series▁TX750▁V2▁<link>▁in▁advance.▁###▁Reply▁2:▁Have▁you▁tried▁crossfire?▁###▁Reply▁3:▁Have▁you▁tried▁dummy▁plugs?I've▁heard▁many▁people▁use▁the▁onboard▁video▁to▁drive▁the▁monitor▁and▁use▁the▁other▁slots▁to▁mine.▁It's▁supposed▁to▁speed▁up▁mining▁as▁doesn't▁have▁to▁draw▁the▁screen.▁###▁Reply▁4:▁I▁did▁2▁seperate▁monitors▁to▁each▁of▁the▁card.▁But▁even▁with▁only▁1▁card▁connected▁to▁the▁second▁PCIe▁slot,▁it▁does▁not▁even▁start▁(fan▁goes▁to▁100%,▁long▁beep▁from▁bios▁and▁black▁monitor).▁###▁Reply▁5:▁Is▁that▁long▁beep▁a▁POST▁code?▁You▁should▁look▁up▁what▁that▁means." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2017-11▁Topic:▁Where▁can▁I▁buy▁ASIC▁antminer▁s9?▁###▁Original▁post:▁HelloI'm▁very▁new▁to▁Bitcoin▁and▁I▁just▁decided▁to▁mine▁Bitcoins▁with▁ASIC▁antminer▁s9.But▁before▁doing▁that,▁I▁need▁to▁buy▁antminer▁s9.After▁a▁big▁investigation,▁I▁see▁there's▁everywhere▁scammers.▁But▁which▁site▁is▁legit▁to▁buy▁antminer▁s9?Thanks▁for▁help▁###▁Reply▁1:▁Bitmain,▁right▁this▁second<link>▁###▁Reply▁2:▁Yeah▁seems▁very▁legit,▁but▁how▁can▁you▁explain▁this?▁<link>▁###▁Reply▁3:▁I'm▁sorry,▁I▁don't▁follow.▁Which▁part▁needs▁explaining?This▁must▁be▁a▁little▁old▁as▁Bitmain▁only▁accepts▁Bitcoin▁Cash▁right▁now.▁###▁Reply▁4:▁The▁only▁trusted▁place▁to▁buy▁S9▁is▁from▁Bitmain.com▁unless▁you▁know▁whom▁you▁are▁dealing▁with▁a▁reputable▁data▁center▁or▁company.▁also▁make▁sure▁that▁they▁will▁take▁care▁of▁the▁warranty▁and▁support▁tickets▁with▁bitmain.▁###▁Reply▁5:▁me▁too!▁###▁Reply▁6:▁thanks▁for▁explaining▁guys!I'm▁gonna▁buy▁an▁ASIC▁antminer▁s9" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0000001\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2014-05▁Topic:▁How▁to▁convert▁.Conf▁file▁into▁.Bat▁###▁Original▁post:▁Hello,▁does▁anybody▁here▁has▁any▁clue▁as▁to▁how▁to▁achieve▁this▁for▁CGminer?▁I▁just▁like▁using▁a▁.bat▁file▁better,▁I'm▁more▁used▁to▁it.▁###▁Reply▁1:▁Don't▁you▁just▁remain▁it▁.bat▁and▁open▁it▁using▁'command▁prompt'▁in▁windows?▁You▁should▁also▁'save▁as'▁if▁you▁are▁using▁notepad▁it▁edit▁the▁text.▁###▁Reply▁2:▁I▁know▁but▁the▁text▁from▁a▁.conf▁file▁looks▁a▁lot▁different▁than▁a▁.bat▁file,▁is▁there▁any▁way▁to▁convert▁one▁without▁having▁to▁edit▁the▁text▁manually?▁###▁Reply▁3:▁try▁it▁another▁waycreate▁bat▁file▁(in▁folder▁with▁cgminer.exe)▁and▁put▁in:▁cgminer▁-c▁path_to_conf_file▁e.g.▁###▁Reply▁4:▁Just▁use▁the▁command▁line▁options▁that▁are▁in▁the▁Top▁Post▁and/or▁readme▁file.▁Easy▁Peasy." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 0.99999994\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2011-08▁Topic:▁The▁community▁needs▁good▁Crossfire-Friendly▁(or,▁exploitive)▁Mining▁Software▁###▁Original▁post:▁The▁community▁needs▁good▁Crossfire-Friendly▁(or,▁exploitive)▁Mining▁Software.......and▁here's▁why▁I▁think▁so:With▁the▁current▁price▁of▁Video▁Cards▁vs.▁the▁current▁ROI,▁it▁really▁doesn't▁make▁sense▁to▁mine▁for▁profit▁using▁a▁6990▁@▁$700+USD▁<truncated>▁3x▁or▁4x▁5770▁Cards▁@▁$99/each▁and▁combine▁their▁solo-200MH/s▁power▁into▁a▁single▁hashing▁POWERHOUSE▁thread▁and▁you▁will▁have▁alot▁of▁people▁saying▁69-what▁?▁....exactly.I▁use▁5770▁cards▁as▁an▁example▁because▁they▁are▁plentiful▁on▁the▁market,▁give▁a▁great▁ROI▁(return▁on▁investment)▁and▁are▁Crossfire▁capable....not▁to▁mention,▁100w/card.Doesn't▁this▁make▁sense▁?1x▁6990▁@▁700MH/s▁=▁$700+USD▁.......OR........▁4x▁5770▁@▁800MH/s▁=▁$400▁USD.The▁money▁you▁would▁save▁on▁power▁supply▁requirements▁would▁go▁directly▁to▁a▁motherboard▁that▁could▁support▁4▁cards.▁###▁Reply▁1:▁you▁can▁mine▁with▁multiple▁cards▁on▁the▁same▁computer▁without▁using▁crossfire▁at▁205-215mh▁each▁i'm▁getting▁400mh▁out▁of▁one▁5770▁equiped▁machine▁i▁bought▁used▁and▁installed▁a▁pair▁of▁them▁in.▁###▁Reply▁2:▁....thanks▁Captain▁Obvious.The▁point▁I▁was▁trying▁to▁make▁was▁to▁take▁4▁shitty▁processing▁threads▁and▁team▁them▁up▁into▁1▁SUPER▁THREAD,▁so▁during▁SHORT▁ROUNDS▁IN▁A▁POOL▁your▁200Mh/s▁cards▁are▁struggling▁to▁make▁2▁shares/each▁when▁the▁6990▁users▁have▁50+▁under▁their▁belts▁in▁the▁same▁amount▁of▁time.GET▁IT▁NOW▁?▁###▁Reply▁3:▁no▁not▁really▁there's▁no▁diffrence▁between▁what▁my▁2▁5770▁cards▁and▁my▁1▁5870▁turn▁in▁when▁they're▁both▁running▁at▁400mh(combined▁or▁alone)▁regardless▁of▁a▁pools▁round▁length.▁###▁Reply▁4:▁that▁only▁applies▁on▁rounds▁that▁are▁like▁500▁shares▁and▁whoever▁solves▁their▁share▁first▁gets▁the▁reward▁and▁everyone▁else▁fails▁###▁Reply▁5:▁crossfire▁wont▁matter▁if▁anything▁it▁would▁slow▁it▁down▁i▁used▁to▁run▁boinc▁and▁it▁did▁work▁in▁crossfire▁and▁it▁would▁slower▁then▁if▁both▁cards▁ran▁work▁units▁alot▁because▁they▁have▁to▁stay▁in▁sync▁if▁one▁card▁goes▁faster▁the▁other▁one▁has▁to▁slow▁down▁or▁stop▁to▁stay▁in▁sync▁with▁it.▁6990▁and▁5970▁cards▁have▁to▁run▁2▁clients▁on▁mining▁just▁like▁having▁2▁real▁video▁cards.▁###▁Reply▁6:▁next▁time▁you▁think▁to▁take▁a▁condescending▁tone▁with▁people▁on▁the▁internet,▁you▁might▁want▁to▁double-check▁that▁what▁you're▁saying▁makes▁sense▁first" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(2048,)\n", + "0.0 1.0000001\n" + ] + }, + { + "data": { + "text/html": [ + "▁Date:▁2015-09▁Topic:▁Uses▁for▁an▁old▁block▁eruptor?▁###▁Original▁post:▁So▁I▁have▁a▁load▁of▁old▁333mh/s▁block▁eruptors,▁using▁some▁on▁the▁lottery,▁but▁the▁question▁is...What▁the▁tits▁do▁I▁do▁with▁10▁333mh/s▁block▁eruptors....▁###▁Reply▁1:▁Sell▁them▁and▁buy▁Sidehack's▁USB▁miners▁to▁lotto▁mine<link>▁###▁Reply▁2:▁List▁them▁on▁eBay▁as▁collectors▁gear.▁More▁for▁owning▁it▁and▁displaying▁it.Then▁use▁the▁money▁to▁buy▁side▁hack▁usb▁sticks▁###▁Reply▁3:▁And▁if▁you▁can't▁manage▁to▁sell▁them,▁they▁should▁make▁good▁\"conversation▁piece\"▁paperweights.▁###▁Reply▁4:▁Heating▁devices,▁they▁works▁extremely▁well▁if▁you▁pair▁them▁with▁some▁quiet▁fans.▁Someone▁even▁install▁them▁in▁the▁central▁heating▁chamber▁to▁send▁warm▁air▁into▁the▁whole▁house▁###▁Reply▁5:▁I▁was▁going▁to▁make▁one▁into▁a▁necklace▁but▁can't▁bring▁myself▁to▁drilling▁into▁it.▁But▁yes,▁put▁them▁on▁Ebay▁someone▁will▁buy▁them.▁###▁Reply▁6:▁I▁would▁agree▁with▁selling.▁They▁have▁a▁surprising▁amount▁of▁value▁still.▁Also▁people▁seem▁to▁collect▁the▁entire▁color▁selection.▁So▁if▁you▁can▁sell▁an▁entire▁set▁depending▁on▁your▁colors▁value▁might▁go▁up.▁###▁Reply▁7:▁You▁don't▁have▁a▁\"load\".▁I▁have▁70.▁That's▁a▁load.And▁guess▁what?▁You▁can't▁do▁anything▁productive▁with▁them.▁They▁are▁worthless.▁The▁\"lottery\"▁is▁a▁waste▁of▁USB▁slots.▁###▁Reply▁8:▁That's▁just▁great!▁If▁you▁live▁near▁the▁Arctic▁Circle.▁It's▁August...▁Summer▁for▁90%▁of▁the▁people▁on▁earth.▁###▁Reply▁9:▁Y▁u▁so▁salty▁bruh...▁I▁put▁my▁s5▁in▁my▁vents▁when▁it▁was▁-3▁celsius▁and▁my▁house▁legit▁warmed▁up▁2▁celsius▁###▁Reply▁10:▁And▁what▁happens▁when▁you▁do▁that▁in▁August?Put▁your▁miners▁in▁your▁vents▁during▁the▁winter!▁But▁now,▁it's▁not▁very▁productive.▁Right?▁Just▁sayin'.▁###▁Reply▁11:▁Anyone▁had▁good▁luck▁recently▁with▁these▁lotto▁devices?▁###▁Reply▁12:▁If▁you▁see▁my▁signature,▁that's▁what▁i've▁been▁doing▁with▁my▁dead▁ones.▁###▁Reply▁13:▁Why▁drill?▁There's▁already▁two▁holes▁going▁through▁the▁heat▁spreaders▁and▁the▁boards,▁one▁conveniently▁near▁a▁corner.▁###▁Reply▁14:▁I▁have▁seen▁them▁on▁eBay▁using▁the▁heat▁sink▁as▁a▁key▁ring." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# visualize the model's predictions\n", + "model = model.to('cpu')\n", + "ds = val_dataset\n", + "# ds = train_dataset\n", + "# id = 0\n", + "for id in range(10):\n", + " with torch.no_grad():\n", + " input_ids = ds[id]['input_ids'].unsqueeze(0)\n", + " attention_mask = ds[id]['attention_mask'].unsqueeze(0)\n", + " labels = ds[id]['labels'].unsqueeze(0)\n", + " outputs = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels)\n", + " logits = outputs.logits\n", + " # print(logits.shape)\n", + "\n", + " predictions = torch.argmax(logits, dim=-1)\n", + " import torch.nn.functional as F\n", + " predictions = F.sigmoid(logits)[0,:,2].numpy()\n", + " print(predictions.shape)\n", + " from sklearn.preprocessing import MinMaxScaler\n", + " scaler = MinMaxScaler()\n", + " predictions = scaler.fit_transform(predictions.reshape(-1, 1)).reshape(-1)\n", + " print(predictions.min(), predictions.max())\n", + "\n", + "\n", + " tokens = tokenizer.convert_ids_to_tokens(input_ids[0].numpy())\n", + " arr = [(token, label) for token, label in zip(tokens, predictions)]\n", + " display(HTML(generate_colored_html(arr[1:])))\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HTML file 'validation_samples.html' has been created.\n" + ] + } + ], + "source": [ + "# First, let's define a function to generate the HTML content for each sample\n", + "def generate_sample_html(sample_id, tokens, predictions, labels):\n", + " html_content = f'

Sample ID: {sample_id}

\\n'\n", + " html_content += '

Predictions

\\n'\n", + " html_content += generate_colored_html([(token, pred) for token, pred in zip(tokens, predictions)][1:])\n", + " html_content += '

Labels

\\n'\n", + " html_content += generate_colored_html([(token, label) for token, label in zip(tokens, labels)][1:])\n", + " return html_content\n", + "\n", + "# Now, let's iterate over the validation dataset and generate HTML for each sample\n", + "html_output = '\\n\\n\\n\\n\\n
\\n'\n", + "\n", + "model = model.cuda()\n", + "\n", + "for id in range(len(val_dataset)):\n", + " with torch.no_grad():\n", + " input_ids = val_dataset[id]['input_ids'].unsqueeze(0)\n", + " attention_mask = val_dataset[id]['attention_mask'].unsqueeze(0)\n", + " labels = val_dataset[id]['labels'].unsqueeze(0)\n", + " outputs = model(input_ids=input_ids.cuda(), attention_mask=attention_mask.cuda())\n", + " logits = outputs.logits\n", + " predictions = torch.argmax(logits, dim=-1)\n", + " predictions = F.sigmoid(logits)[0,:,2].cpu().numpy()\n", + " # scaler = MinMaxScaler()\n", + " # predictions = scaler.fit_transform(predictions.reshape(-1, 1)).reshape(-1)\n", + " predictions *= 2 # Adjusting prediction values for visualization\n", + " tokens = tokenizer.convert_ids_to_tokens(input_ids[0].numpy())\n", + " labels = labels[0].numpy() * 0.5 # Adjusting label values for visualization\n", + " html_output += generate_sample_html(id, tokens, predictions, labels).replace(\"▁\", \" \").replace(\"###\", \"
###\")\n", + " html_output += '
\\n'\n", + " if id == 50:\n", + " break\n", + "\n", + "html_output += '
\\n\\n'\n", + "\n", + "# Finally, save the HTML content to a file\n", + "with open('validation_samples.html', 'w') as f:\n", + " f.write(html_output)\n", + "\n", + "print(\"HTML file 'validation_samples.html' has been created.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.1.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}