<reponame>SimeoW/bitcoin_researcher<filename>tools/Dynamic Block Delay Reputation Controller/Train_Some_More.py
from statistics import median
import csv
import json
import os
import random
import re
import sys
import time


# Path to the folder containing all the .csv files
# Generated from the logger: https://github.com/simewu/bitcoin_researcher/blob/master/tools/LogAlternatingConnections.py
datasetPath = 'Z:\\Simeon\\OneDrive - University of Colorado Colorado Springs\\!!! School\\2020\\Research\\Public Node, Maximizing Connections\\Computing rates (1 to 512 peers)\\PostProcessed'

# Index to the column that contains the list of addresses connected to
columnIndexToConnectionsList = 10
# Index to the column that contains the block [and header] propagation times
columnIndexToBlockAndHeaderDelay = 17

global globalReputationJson
globalReputationJson = {
	# How many random files it has been sampled and trained off of
	'trainedCounter': 0,

	# A score threshold over all files so we can determine if any particular sample is good (>), bad (<), or neutral (==)
	# Median block propagation time in seconds
	'sentimentScoreThreshold': None,

	# A key (user IPv4, IPv6, or Tor address) and value (unbounded reputation of that node)
	'dictionary': {},
}


# Main application loop
def main():
	global globalReputationJson
	restoreGlobalReputationDictionaryCache()

	if globalReputationJson['sentimentScoreThreshold'] is None:
		print()
		print('Sentiment score threshold needs to first be computed.')
		print('This will loop through all files in the dataset and compute the median threshold.')
		print('This may take a long time.')
		proceed = input('Proceed? (y/n):').lower() in ['y', 'yes']

		if not proceed:
			print('Goodbye.')
			sys.exit()

		# Go through every file and get its average score, then take note of the median score
		computeSentimentScoreThreshold()

		print('Sentiment score threshold has been computed!')
		print('Beginning training in 10 seconds...')
		time.sleep(10)

	datasetFiles = listFiles('.*\.csv', datasetPath)
	counter = 0
	try:
		while True:
			time.sleep(0.1)
			sampleFilePath = random.choice(datasetFiles)
			iterateOnce(sampleFilePath)

			counter += 1
			if counter % 10 == 0:
				saveGlobalReputationDictionaryCache()

	except KeyboardInterrupt:
		pass

	print('Goodbye.')


# Given a sample file's path, attempt to train off of it (if it has a score), apply it to the global reputation dictionary, then return
def iterateOnce(sampleFilePath):
	global globalReputationJson

	print(f'Iteration #{globalReputationJson["trainedCounter"] + 1}, processing "{sampleFilePath}"...')
	readerFile = open(sampleFilePath, 'r')
	# Remove NUL bytes to prevent errors
	reader = csv.reader(x.replace('\0', '') for x in readerFile)
	header = next(reader)
	headerDesc = next(reader)

	averageScore = 0
	averageScoreCount = 0
	nodesConnectedTo = []

	score = None
	prevScore = None

	# Compute the average score of this sample
	for row in reader:
		prevScore = score
		score = getScoringProperty(row)

		if score is None:
			# No score available
			# Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time
			continue

		if score == prevScore:
			# This row in the file provides no new information (same score as the row before it)
			continue

		averageScore += score
		averageScoreCount += 1

		# Build up a list of all connections that we've had in this sample file
		nodesConnectedTo = unionWithoutRepetition(nodesConnectedTo, getListOfNodeConnections(row))

	if averageScoreCount == 0:
		# No scores were found, invalid sample file
		print(f'No score information available for "{sampleFilePath}", skipping')
		return

	averageScore /= averageScoreCount

	# Decide the sentiment of this average score value, good or bad
	decision = 0
	if averageScore > globalReputationJson['sentimentScoreThreshold']:
		decision = 1 / len(nodesConnectedTo)
	elif averageScore < globalReputationJson['sentimentScoreThreshold']:
		decision = -1 / len(nodesConnectedTo)

	# Apply our decision to the global node reputations
	for node in nodesConnectedTo:
		if node not in globalReputationJson:
			# Create the data entry if it doesn't exist
			globalReputationJson['dictionary'][node] = [decision, 1]
		else:
			# Otherwise increment (or decrement) the score
			globalReputationJson['dictionary'][node][0] += decision
			globalReputationJson['dictionary'][node][1] += 1

	globalReputationJson['trainedCounter'] += 1
	readerFile.close()

	# Done
	return


# List the files with a regular expression
def listFiles(regex, directory = ''):
	path = os.path.join(os.curdir, directory)
	return [os.path.join(path, file) for file in os.listdir(path) if os.path.isfile(os.path.join(path, file)) and bool(re.match(regex, file))]

# Merge two lists into one without allowing any duplicates
# Used to combine duplicate IP addresses into one list
def unionWithoutRepetition(list1, list2):
	return list(set(list1 + list2))

# Given a row from the dataset, returns the corresponding block delay (score), or None if it doesn't exist
def getScoringProperty(row):
	val = row[columnIndexToBlockAndHeaderDelay]
	if '/' in val:
		# Since this cell contains both block delay and header delay
		val = val.split('/')[0]

	val = val.strip() # Remove any whitespace

	if val == '': return None
	return float(val)

# Given a row from the dataset, returns a list of (IPv4, IPv6, Tor) addresses not including the port
def getListOfNodeConnections(row):
	nodes = []
	rawNodes = row[columnIndexToConnectionsList].split(' ')
	for rawNode in rawNodes:
		fragments = rawNode.split(':')
		if len(fragments) > 1:
			# Remove the port number from the IP address
			fragments.pop()
		# Put all the IP address fragments back together
		node = ':'.join(fragments)
		nodes.append(node)
	return nodes

# Go through every file and get its average score, then take note of the median score
def computeSentimentScoreThreshold():
	global globalReputationJson
	averagedScores = []

	datasetFiles = listFiles('.*\.csv', datasetPath)
	for i, fileName in enumerate(datasetFiles):
		print(f'Updating sentiment score threshold ({i + 1} / {len(datasetFiles)})')

		readerFile = open(fileName, 'r')
		# Remove NUL bytes to prevent errors
		reader = csv.reader(x.replace('\0', '') for x in readerFile)
		header = next(reader)
		headerDesc = next(reader)

		prevScore = None
		score = None

		averageScore = 0
		averageScoreCount = 0

		# Compute the average score of this file
		for row in reader:
			prevScore = score
			score = getScoringProperty(row)

			if score is None:
				# No score available
				# Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time
				continue

			if score == prevScore:
				# This row in the file provides no new information (same score as the row before it)
				continue

			averageScore += score
			averageScoreCount += 1

		if averageScoreCount == 0:
			# No scores were found, invalid sample file
			continue

		averageScore /= averageScoreCount

		averagedScores.append(averageScore)

		readerFile.close()
	
	globalReputationJson['sentimentScoreThreshold'] = median(averagedScores)
	saveGlobalReputationDictionaryCache()




# Save our trained data to file
def saveGlobalReputationDictionaryCache():
	global globalReputationJson
	print('Saving global reputation JSON dictionary...')

	with open('globalReputationDictionaryCache.json', 'w') as outputFile:
		json.dump(globalReputationJson, outputFile)

# Restore our trained data from file
def restoreGlobalReputationDictionaryCache():
	global globalReputationJson
	print('Restoring global reputation JSON dictionary...')

	# If it doesn't exist, let's save the blank structure
	if not os.path.exists('globalReputationDictionaryCache.json'):
		saveGlobalReputationDictionaryCache()
		return

	with open('globalReputationDictionaryCache.json') as inputFile:
		globalReputationJson = json.load(inputFile)

if __name__ == '__main__':
	main()

Block Delay Reputation Controller/Train_Some_More.py
from statistics import median
import csv
import json
import os
import random
import re
import sys
import time


# Path to the folder containing all the .csv files
# Generated from the logger: https://github.com/simewu/bitcoin_researcher/blob/master/tools/LogAlternatingConnections.py
datasetPath = 'Z:\\Simeon\\OneDrive - University of Colorado Colorado Springs\\!!! School\\2020\\Research\\Public Node, Maximizing Connections\\Computing rates (1 to 512 peers)\\PostProcessed'

# Index to the column that contains the list of addresses connected to
columnIndexToConnectionsList = 10
# Index to the column that contains the block [and header] propagation times
columnIndexToBlockAndHeaderDelay = 17

global globalReputationJson
globalReputationJson = {
	# How many random files it has been sampled and trained off of
	'trainedCounter': 0,

	# A score threshold over all files so we can determine if any particular sample is good (>), bad (<), or neutral (==)
	# Median block propagation time in seconds
	'sentimentScoreThreshold': None,

	# A key (user IPv4, IPv6, or Tor address) and value (unbounded reputation of that node)
	'dictionary': {},
}


# Main application loop
def main():
	global globalReputationJson
	restoreGlobalReputationDictionaryCache()

	if globalReputationJson['sentimentScoreThreshold'] is None:
		print()
		print('Sentiment score threshold needs to first be computed.')
		print('This will loop through all files in the dataset and compute the median threshold.')
		print('This may take a long time.')
		proceed = input('Proceed? (y/n):').lower() in ['y', 'yes']

		if not proceed:
			print('Goodbye.')
			sys.exit()

		# Go through every file and get its average score, then take note of the median score
		computeSentimentScoreThreshold()

		print('Sentiment score threshold has been computed!')
		print('Beginning training in 10 seconds...')
		time.sleep(10)

	datasetFiles = listFiles('.*\.csv', datasetPath)
	counter = 0
	try:
		while True:
			time.sleep(0.1)
			sampleFilePath = random.choice(datasetFiles)
			iterateOnce(sampleFilePath)

			counter += 1
			if counter % 10 == 0:
				saveGlobalReputationDictionaryCache()

	except KeyboardInterrupt:
		pass

	print('Goodbye.')


# Given a sample file's path, attempt to train off of it (if it has a score), apply it to the global reputation dictionary, then return
def iterateOnce(sampleFilePath):
	global globalReputationJson

	print(f'Iteration #{globalReputationJson["trainedCounter"] + 1}, processing "{sampleFilePath}"...')
	readerFile = open(sampleFilePath, 'r')
	# Remove NUL bytes to prevent errors
	reader = csv.reader(x.replace('\0', '') for x in readerFile)
	header = next(reader)
	headerDesc = next(reader)

	averageScore = 0
	averageScoreCount = 0
	nodesConnectedTo = []

	score = None
	prevScore = None

	# Compute the average score of this sample
	for row in reader:
		prevScore = score
		score = getScoringProperty(row)

		if score is None:
			# No score available
			# Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time
			continue

		if score == prevScore:
			# This row in the file provides no new information (same score as the row before it)
			continue

		averageScore += score
		averageScoreCount += 1

		# Build up a list of all connections that we've had in this sample file
		nodesConnectedTo = unionWithoutRepetition(nodesConnectedTo, getListOfNodeConnections(row))

	if averageScoreCount == 0:
		# No scores were found, invalid sample file
		print(f'No score information available for "{sampleFilePath}", skipping')
		return

	averageScore /= averageScoreCount

	# Decide the sentiment of this average score value, good or bad
	decision = 0
	if averageScore > globalReputationJson['sentimentScoreThreshold']:
		decision = 1 / len(nodesConnectedTo)
	elif averageScore < globalReputationJson['sentimentScoreThreshold']:
		decision = -1 / len(nodesConnectedTo)

	# Apply our decision to the global node reputations
	for node in nodesConnectedTo:
		if node not in globalReputationJson:
			# Create the data entry if it doesn't exist
			globalReputationJson['dictionary'][node] = [decision, 1]
		else:
			# Otherwise increment (or decrement) the score
			globalReputationJson['dictionary'][node][0] += decision
			globalReputationJson['dictionary'][node][1] += 1

	globalReputationJson['trainedCounter'] += 1
	readerFile.close()

	# Done
	return


# List the files with a regular expression
def listFiles(regex, directory = ''):
	path = os.path.join(os.curdir, directory)
	return [os.path.join(path, file) for file in os.listdir(path) if os.path.isfile(os.path.join(path, file)) and bool(re.match(regex, file))]

# Merge two lists into one without allowing any duplicates
# Used to combine duplicate IP addresses into one list
def unionWithoutRepetition(list1, list2):
	return list(set(list1 + list2))

# Given a row from the dataset, returns the corresponding block delay (score), or None if it doesn't exist
def getScoringProperty(row):
	val = row[columnIndexToBlockAndHeaderDelay]
	if '/' in val:
		# Since this cell contains both block delay and header delay
		val = val.split('/')[0]

	val = val.strip() # Remove any whitespace

	if val == '': return None
	return float(val)

# Given a row from the dataset, returns a list of (IPv4, IPv6, Tor) addresses not including the port
def getListOfNodeConnections(row):
	nodes = []
	rawNodes = row[columnIndexToConnectionsList].split(' ')
	for rawNode in rawNodes:
		fragments = rawNode.split(':')
		if len(fragments) > 1:
			# Remove the port number from the IP address
			fragments.pop()
		# Put all the IP address fragments back together
		node = ':'.join(fragments)
		nodes.append(node)
	return nodes

# Go through every file and get its average score, then take note of the median score
def computeSentimentScoreThreshold():
	global globalReputationJson
	averagedScores = []

	datasetFiles = listFiles('.*\.csv', datasetPath)
	for i, fileName in enumerate(datasetFiles):
		print(f'Updating sentiment score threshold ({i + 1} / {len(datasetFiles)})')

		readerFile = open(fileName, 'r')
		# Remove NUL bytes to prevent errors
		reader = csv.reader(x.replace('\0', '') for x in readerFile)
		header = next(reader)
		headerDesc = next(reader)

		prevScore = None
		score = None

		averageScore = 0
		averageScoreCount = 0

		# Compute the average score of this file
		for row in reader:
			prevScore = score
			score = getScoringProperty(row)

			if score is None:
				# No score available
				# Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time
				continue

			if score == prevScore:
				# This row in the file provides no new information (same score as the row before it)
				continue

			averageScore += score
			averageScoreCount += 1

		if averageScoreCount == 0:
			# No scores were found, invalid sample file
			continue

		averageScore /= averageScoreCount

		averagedScores.append(averageScore)

		readerFile.close()
	
	globalReputationJson['sentimentScoreThreshold'] = median(averagedScores)
	saveGlobalReputationDictionaryCache()




# Save our trained data to file
def saveGlobalReputationDictionaryCache():
	global globalReputationJson
	print('Saving global reputation JSON dictionary...')

	with open('globalReputationDictionaryCache.json', 'w') as outputFile:
		json.dump(globalReputationJson, outputFile)

# Restore our trained data from file
def restoreGlobalReputationDictionaryCache():
	global globalReputationJson
	print('Restoring global reputation JSON dictionary...')

	# If it doesn't exist, let's save the blank structure
	if not os.path.exists('globalReputationDictionaryCache.json'):
		saveGlobalReputationDictionaryCache()
		return

	with open('globalReputationDictionaryCache.json') as inputFile:
		globalReputationJson = json.load(inputFile)

if __name__ == '__main__':
	main()
# Path to the folder containing all the .csv files # Generated from the logger: https://github.com/simewu/bitcoin_researcher/blob/master/tools/LogAlternatingConnections.py # Index to the column that contains the list of addresses connected to # Index to the column that contains the block [and header] propagation times # How many random files it has been sampled and trained off of # A score threshold over all files so we can determine if any particular sample is good (>), bad (<), or neutral (==) # Median block propagation time in seconds # A key (user IPv4, IPv6, or Tor address) and value (unbounded reputation of that node) # Main application loop # Go through every file and get its average score, then take note of the median score # Given a sample file's path, attempt to train off of it (if it has a score), apply it to the global reputation dictionary, then return #{globalReputationJson["trainedCounter"] + 1}, processing "{sampleFilePath}"...') # Remove NUL bytes to prevent errors # Compute the average score of this sample # No score available # Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time # This row in the file provides no new information (same score as the row before it) # Build up a list of all connections that we've had in this sample file # No scores were found, invalid sample file # Decide the sentiment of this average score value, good or bad # Apply our decision to the global node reputations # Create the data entry if it doesn't exist # Otherwise increment (or decrement) the score # Done # List the files with a regular expression # Merge two lists into one without allowing any duplicates # Used to combine duplicate IP addresses into one list # Given a row from the dataset, returns the corresponding block delay (score), or None if it doesn't exist # Since this cell contains both block delay and header delay # Remove any whitespace # Given a row from the dataset, returns a list of (IPv4, IPv6, Tor) addresses not including the port # Remove the port number from the IP address # Put all the IP address fragments back together # Go through every file and get its average score, then take note of the median score # Remove NUL bytes to prevent errors # Compute the average score of this file # No score available # Happens the first few rows when we haven't received a new block yet, so we don't know the block prop time # This row in the file provides no new information (same score as the row before it) # No scores were found, invalid sample file # Save our trained data to file # Restore our trained data from file # If it doesn't exist, let's save the blank structure