import re
from os import listdir, mkdir
from shutil import rmtree
from os.path import isfile, join, expanduser
import binascii
import sys
import requests
import re

baseDlDir = expanduser('~/github/gh.MalwareAnalysisTools/tempdl')


def findAllEncodedUrls(path):
	af = ""
	with open(path, "rb") as f:
		af = f.read()

	allUrls = list()

	for m in re.finditer(br'(?P<enc>6800740074007000([0-9a-fA-F][0-9a-fA-F]00)+?)(00|20)00', af):
		allUrls.append(binascii.unhexlify(m['enc'].replace(b'00', b'')).decode('ascii'))

	return allUrls


visitedUrls = dict()
visitedUrlsInd = 0


def processFile(path):
	global visitedUrlsInd, visitedUrls

	print('Processing file {}'.format(path))
	results = dict({'path': path})

	results['rtfUrls'] = findAllEncodedUrls(path)

	for u in results['rtfUrls']:
		print('... {}'.format(u), end='')

		if u not in visitedUrls.values():
			visitedUrls[visitedUrlsInd] = u
			visitedUrlsInd = visitedUrlsInd+1
			print(': NEW')
		else:
			print(': DUP')

	return results


def cleanDlPath(path):
	try:
		rmtree(path)
	except FileNotFoundError:
		pass
	mkdir(path)


def downloadUrls(urls, basePath):
	dlContent = list()

	for i in urls:
		res = dict({'ind': i, 'url': urls[i]})

		print('({}/{} ProcessUrl ({}): '.format(len(dlContent)+1, len(urls), res['url']), end='')

		(shouldDownload, reason) = shouldDownloadUrl(urls[i])

		if not shouldDownload:
			res['status'] = 'SKIP'
			res['error'] = reason
		else:
			try:
				resp = requests.get(res['url'], timeout=5)

				if resp.ok:
					p = join(basePath, str(res['ind']))
					with open(p, 'wb') as f:
						f.write(resp.content)
					res['status'] = 'OK'
				else:
					res['status'] = 'ERROR'
					res['error'] = 'status_code({})'.format(resp.status_code)

			except Exception as e:
				res['error'] = e
				res['status'] = 'EXCEPT'
				pass

		if res['status'] != 'OK':
			print('{}: {}'.format(res['status'], res['error']))
		else:
			print('{}'.format(res['status']))

		dlContent.append(res)
	return dlContent


def shouldDownloadUrl(url):
	dontDownloadRes = [
		(re.compile(r'https?://192\.168\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://172\.0?(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://[a-z0-9.-]*[a-z][a-z0-9.-]*/?$'), 'Domain Root')
	]

	for r, e in dontDownloadRes:
		if r.match(url):
			return False\
				, e
	return True, ''

def processDir(path):
	print('Processing path {}'.format(path))
	results = list()
	for f in [f for f in listdir(path) if isfile(join(path, f))]:
		results.append(processFile(join(path, f)))

	return results


def main():
	results = list()
	for a in sys.argv:
		if isfile(a):
			results.append(processFile(a))
		else:
			r2 = processDir(a)

			if r2:
				results = results + r2

	cleanDlPath(baseDlDir)
	aa = downloadUrls(visitedUrls, baseDlDir)
	print(aa)


if __name__ == "__main__":
	main()

import re
from os import listdir, mkdir
from shutil import rmtree
from os.path import isfile, join, expanduser
import binascii
import sys
import requests
import re

baseDlDir = expanduser('~/github/gh.MalwareAnalysisTools/tempdl')


def findAllEncodedUrls(path):
	af = ""
	with open(path, "rb") as f:
		af = f.read()

	allUrls = list()

	for m in re.finditer(br'(?P<enc>6800740074007000([0-9a-fA-F][0-9a-fA-F]00)+?)(00|20)00', af):
		allUrls.append(binascii.unhexlify(m['enc'].replace(b'00', b'')).decode('ascii'))

	return allUrls


visitedUrls = dict()
visitedUrlsInd = 0


def processFile(path):
	global visitedUrlsInd, visitedUrls

	print('Processing file {}'.format(path))
	results = dict({'path': path})

	results['rtfUrls'] = findAllEncodedUrls(path)

	for u in results['rtfUrls']:
		print('... {}'.format(u), end='')

		if u not in visitedUrls.values():
			visitedUrls[visitedUrlsInd] = u
			visitedUrlsInd = visitedUrlsInd+1
			print(': NEW')
		else:
			print(': DUP')

	return results


def cleanDlPath(path):
	try:
		rmtree(path)
	except FileNotFoundError:
		pass
	mkdir(path)


def downloadUrls(urls, basePath):
	dlContent = list()

	for i in urls:
		res = dict({'ind': i, 'url': urls[i]})

		print('({}/{} ProcessUrl ({}): '.format(len(dlContent)+1, len(urls), res['url']), end='')

		(shouldDownload, reason) = shouldDownloadUrl(urls[i])

		if not shouldDownload:
			res['status'] = 'SKIP'
			res['error'] = reason
		else:
			try:
				resp = requests.get(res['url'], timeout=5)

				if resp.ok:
					p = join(basePath, str(res['ind']))
					with open(p, 'wb') as f:
						f.write(resp.content)
					res['status'] = 'OK'
				else:
					res['status'] = 'ERROR'
					res['error'] = 'status_code({})'.format(resp.status_code)

			except Exception as e:
				res['error'] = e
				res['status'] = 'EXCEPT'
				pass

		if res['status'] != 'OK':
			print('{}: {}'.format(res['status'], res['error']))
		else:
			print('{}'.format(res['status']))

		dlContent.append(res)
	return dlContent


def shouldDownloadUrl(url):
	dontDownloadRes = [
		(re.compile(r'https?://192\.168\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://172\.0?(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?/'), 'Private IP'),
		(re.compile(r'https?://[a-z0-9.-]*[a-z][a-z0-9.-]*/?$'), 'Domain Root')
	]

	for r, e in dontDownloadRes:
		if r.match(url):
			return False\
				, e
	return True, ''

def processDir(path):
	print('Processing path {}'.format(path))
	results = list()
	for f in [f for f in listdir(path) if isfile(join(path, f))]:
		results.append(processFile(join(path, f)))

	return results


def main():
	results = list()
	for a in sys.argv:
		if isfile(a):
			results.append(processFile(a))
		else:
			r2 = processDir(a)

			if r2:
				results = results + r2

	cleanDlPath(baseDlDir)
	aa = downloadUrls(visitedUrls, baseDlDir)
	print(aa)


if __name__ == "__main__":
	main()
