#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Convert the *DECOW14X* corpus into a plain text file. Is used as pre-processing step for the
`word2vec <https://code.google.com/archive/p/word2vec/>`_ training.
To make this this more feasible (decow is a **huge** corpus), python's :mod:`multiprocessing` is used, s.t. every
part of the corpus in simultaneously processed. Afterwards, a bash command like ``cat`` can be used to merge into one
single file.
"""

# STANDARD
import codecs
import gzip as gz
import multiprocessing
import optparse
import os
import re

# PROJECT
from src.misc.decorators import log_time
from src.misc.helpers import alt, contains_tag, extract_sentence_id


def main():
	"""
	Main function. Uses command lines to start corpus processing.
	"""
	optparser = optparse.OptionParser()
	optparser.add_option('--in', dest='in_dir', help='Path to input directory')
	optparser.add_option('--out', dest='out', help='Path to output directory')
	optparser.add_option('--merge', dest='merge', action="store_true", help='Merge multi-word named entities?')
	optparser.add_option('--log', dest='log', help='Path to logfile')
	optparser.add_option('--log_interval', dest='inter', type='int', help='Logging interval')
	(options, args) = optparser.parse_args()
	convert_decow_to_plain(options.in_dir, options.out, options.log, options.merge, options.inter)


def convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval):
	"""
	Convert the whole corpus into plain text.

	Args:
		decow_dir (str): Path to directory with decow corpus paths.
		out_dir (str): Path where plain text parts should be written to.
		log_path (str): Path where the log files should be written to.
		merge_nes (bool): Flag to indicate whether multi-word expression should be merged with underscores.
		log_interval (int): Interval to log current process state in seconds.
	"""
	# Split logging interval into hourse - minutes - seconds
	m_proc, s_proc = divmod(log_interval, 60)
	h_proc, m_proc = divmod(m_proc, 60)

	# Init logfile
	with codecs.open(log_path, "a", "utf-8") as log_file:
		log_file.write(alt("Starting logging...\n"))
		log_file.write(alt("Corpus (parts) directory:\t%s\n" % decow_dir))
		log_file.write(alt("Output directory:\t\t%s\n" % out_dir))
		log_file.write(alt("Logging path:\t\t%s\n" % log_path))
		log_file.write(alt("Logging intervals:\n\t Every %2dh %2dm %2ds for metalog\n" % (h_proc, m_proc, s_proc)))

	# Start processes
	@log_time(log_path, log_interval)
	def _convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval):
		with codecs.open(log_path, "a", "utf-8") as log_file:
			log_file.write(alt("Preparing %i process(es)...\n" %(len(decow_dir))))
			inpaths = [path for path in os.listdir(decow_dir) if ".DS_Store" not in path and "decow" in path]
			pool = multiprocessing.Pool(processes=len(inpaths))
			log_file.write(alt("Starting process(es)!\n"))
			if merge_nes:
				pool.map(convert_part_merging, [(decow_dir + inpath, out_dir, log_path, log_interval) for inpath in inpaths])
			else:
				pool.map(convert_part, [(decow_dir + inpath, out_dir, log_path, log_interval) for inpath in inpaths])

	_convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval)


def convert_part(argstuple):
	"""
	Convert a corpus part into plain text without merging multiple word entries.

	Args:
		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath``
					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval``
					(*int*): Logging interval in seconds)
	"""
	inpath, dir_outpath, log_path, interval = argstuple

	@log_time(log_path, interval)
	def _convert_part(inpath, dir_outpath):
		file_n = get_file_number(inpath)
		outpath = dir_outpath + 'decow%s_out.txt' %(str(file_n))
		with gz.open(inpath, 'rb') as infile, codecs.open(outpath, 'wb', 'utf-8') as outfile:
			sentence = []
			for line in infile:
				line = line.strip().decode("utf-8")
				if line.startswith(u'<s'):
					outfile.write('%s\n' %(' '.join(sentence)))
					sentence = []
				if not line.startswith(u'<'):
					sentence.append(line.split('\t')[0])

	_convert_part(inpath, dir_outpath)


def convert_part_merging(argstuple):
	"""
	Convert a corpus part into plain text and merging multiple word entries.

	Args:
		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath``
					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval``
					(*int*): Logging interval in seconds)
	"""
	inpath, dir_outpath, log_path, interval = argstuple

	@log_time(log_path, interval)
	def _convert_part_merging(inpath, dir_outpath, log_path):
		with codecs.open(log_path, "a", "utf-8") as log_file:
			process_name = multiprocessing.current_process().name
			log_file.write(alt("%s: Start logging processing of\n\t%s to \n\t%s...\n" % (process_name, inpath, dir_outpath)))
			file_n = get_file_number(inpath)
			outpath = dir_outpath + 'decow%s_out.txt' %(str(file_n))

			with gz.open(inpath, 'rb') as infile, codecs.open(outpath, 'wb', 'utf-8') as outfile:
				sentence = []
				line, lcount = infile.readline().strip().decode("utf-8"), 1

				while line != "":
					if lcount % 100000 == 0:
						log_file.write(alt("%s: Processing line nr. %i...\n" % (process_name, lcount)))

					ne = extract_named_entity(line)  # Extract possible named entity

					if line.startswith(u'<s'):
						outfile.write('%s\n' %(' '.join(sentence)))
						sentence = []
					# If there was a named entity found, try to complete it if it's a multi-word expression
					elif ne is not None:
						while True:
							next_line = infile.readline().strip().decode("utf-8")
							lcount += 1
							if not contains_tag(next_line):
								next_ne = extract_named_entity(next_line)
								if next_ne is not None and next_ne[1] == ne[1]:
									ne = ("%s_%s" %(ne[0], next_ne[0]), ne[1])
								else:
									break
							else:
								break
						sentence.append(ne[0])
						line = next_line
						continue
					elif not line.startswith(u'<'):
						sentence.append(line.split('\t')[0])
					line, lcount = infile.readline().strip().decode("utf-8"), lcount + 1

	_convert_part_merging(inpath, dir_outpath, log_path)


def get_file_number(filename):
	"""
	Get the number of the current decow corpus part.

	Args:
		filename (str): Decow corpus part file name

	Returns:
		str: File number
	"""
	file_n = re.findall(re.compile("\d{2}(?=[^a])"), filename)  # Retrieve file number
	if len(file_n) == 0:
		file_n = re.findall(re.compile("\d+"), filename)[len(file_n) - 1]
	else:
		file_n = file_n[len(file_n) - 1]
	return file_n if int(file_n) > 9 else "0" + file_n


def extract_named_entity(line):
	"""
	Extract named entity from current line.

	Args:
		line (str): Current line

	Returns:
		str or None: Extracted named entity or None if no named entity is present.
	"""
	try:
		line_parts = line.split("\t")
		feature = line_parts[3]
		if feature != "O":
			return line_parts[2], line_parts[3]
	except IndexError:
		return None


if __name__ == '__main__':
	main()

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Convert the *DECOW14X* corpus into a plain text file. Is used as pre-processing step for the
`word2vec <https://code.google.com/archive/p/word2vec/>`_ training.
To make this this more feasible (decow is a **huge** corpus), python's :mod:`multiprocessing` is used, s.t. every
part of the corpus in simultaneously processed. Afterwards, a bash command like ``cat`` can be used to merge into one
single file.
"""

# STANDARD
import codecs
import gzip as gz
import multiprocessing
import optparse
import os
import re

# PROJECT
from src.misc.decorators import log_time
from src.misc.helpers import alt, contains_tag, extract_sentence_id


def main():
	"""
	Main function. Uses command lines to start corpus processing.
	"""
	optparser = optparse.OptionParser()
	optparser.add_option('--in', dest='in_dir', help='Path to input directory')
	optparser.add_option('--out', dest='out', help='Path to output directory')
	optparser.add_option('--merge', dest='merge', action="store_true", help='Merge multi-word named entities?')
	optparser.add_option('--log', dest='log', help='Path to logfile')
	optparser.add_option('--log_interval', dest='inter', type='int', help='Logging interval')
	(options, args) = optparser.parse_args()
	convert_decow_to_plain(options.in_dir, options.out, options.log, options.merge, options.inter)


def convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval):
	"""
	Convert the whole corpus into plain text.

	Args:
		decow_dir (str): Path to directory with decow corpus paths.
		out_dir (str): Path where plain text parts should be written to.
		log_path (str): Path where the log files should be written to.
		merge_nes (bool): Flag to indicate whether multi-word expression should be merged with underscores.
		log_interval (int): Interval to log current process state in seconds.
	"""
	# Split logging interval into hourse - minutes - seconds
	m_proc, s_proc = divmod(log_interval, 60)
	h_proc, m_proc = divmod(m_proc, 60)

	# Init logfile
	with codecs.open(log_path, "a", "utf-8") as log_file:
		log_file.write(alt("Starting logging...\n"))
		log_file.write(alt("Corpus (parts) directory:\t%s\n" % decow_dir))
		log_file.write(alt("Output directory:\t\t%s\n" % out_dir))
		log_file.write(alt("Logging path:\t\t%s\n" % log_path))
		log_file.write(alt("Logging intervals:\n\t Every %2dh %2dm %2ds for metalog\n" % (h_proc, m_proc, s_proc)))

	# Start processes
	@log_time(log_path, log_interval)
	def _convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval):
		with codecs.open(log_path, "a", "utf-8") as log_file:
			log_file.write(alt("Preparing %i process(es)...\n" %(len(decow_dir))))
			inpaths = [path for path in os.listdir(decow_dir) if ".DS_Store" not in path and "decow" in path]
			pool = multiprocessing.Pool(processes=len(inpaths))
			log_file.write(alt("Starting process(es)!\n"))
			if merge_nes:
				pool.map(convert_part_merging, [(decow_dir + inpath, out_dir, log_path, log_interval) for inpath in inpaths])
			else:
				pool.map(convert_part, [(decow_dir + inpath, out_dir, log_path, log_interval) for inpath in inpaths])

	_convert_decow_to_plain(decow_dir, out_dir, log_path, merge_nes, log_interval)


def convert_part(argstuple):
	"""
	Convert a corpus part into plain text without merging multiple word entries.

	Args:
		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath``
					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval``
					(*int*): Logging interval in seconds)
	"""
	inpath, dir_outpath, log_path, interval = argstuple

	@log_time(log_path, interval)
	def _convert_part(inpath, dir_outpath):
		file_n = get_file_number(inpath)
		outpath = dir_outpath + 'decow%s_out.txt' %(str(file_n))
		with gz.open(inpath, 'rb') as infile, codecs.open(outpath, 'wb', 'utf-8') as outfile:
			sentence = []
			for line in infile:
				line = line.strip().decode("utf-8")
				if line.startswith(u'<s'):
					outfile.write('%s\n' %(' '.join(sentence)))
					sentence = []
				if not line.startswith(u'<'):
					sentence.append(line.split('\t')[0])

	_convert_part(inpath, dir_outpath)


def convert_part_merging(argstuple):
	"""
	Convert a corpus part into plain text and merging multiple word entries.

	Args:
		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath``
					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval``
					(*int*): Logging interval in seconds)
	"""
	inpath, dir_outpath, log_path, interval = argstuple

	@log_time(log_path, interval)
	def _convert_part_merging(inpath, dir_outpath, log_path):
		with codecs.open(log_path, "a", "utf-8") as log_file:
			process_name = multiprocessing.current_process().name
			log_file.write(alt("%s: Start logging processing of\n\t%s to \n\t%s...\n" % (process_name, inpath, dir_outpath)))
			file_n = get_file_number(inpath)
			outpath = dir_outpath + 'decow%s_out.txt' %(str(file_n))

			with gz.open(inpath, 'rb') as infile, codecs.open(outpath, 'wb', 'utf-8') as outfile:
				sentence = []
				line, lcount = infile.readline().strip().decode("utf-8"), 1

				while line != "":
					if lcount % 100000 == 0:
						log_file.write(alt("%s: Processing line nr. %i...\n" % (process_name, lcount)))

					ne = extract_named_entity(line)  # Extract possible named entity

					if line.startswith(u'<s'):
						outfile.write('%s\n' %(' '.join(sentence)))
						sentence = []
					# If there was a named entity found, try to complete it if it's a multi-word expression
					elif ne is not None:
						while True:
							next_line = infile.readline().strip().decode("utf-8")
							lcount += 1
							if not contains_tag(next_line):
								next_ne = extract_named_entity(next_line)
								if next_ne is not None and next_ne[1] == ne[1]:
									ne = ("%s_%s" %(ne[0], next_ne[0]), ne[1])
								else:
									break
							else:
								break
						sentence.append(ne[0])
						line = next_line
						continue
					elif not line.startswith(u'<'):
						sentence.append(line.split('\t')[0])
					line, lcount = infile.readline().strip().decode("utf-8"), lcount + 1

	_convert_part_merging(inpath, dir_outpath, log_path)


def get_file_number(filename):
	"""
	Get the number of the current decow corpus part.

	Args:
		filename (str): Decow corpus part file name

	Returns:
		str: File number
	"""
	file_n = re.findall(re.compile("\d{2}(?=[^a])"), filename)  # Retrieve file number
	if len(file_n) == 0:
		file_n = re.findall(re.compile("\d+"), filename)[len(file_n) - 1]
	else:
		file_n = file_n[len(file_n) - 1]
	return file_n if int(file_n) > 9 else "0" + file_n


def extract_named_entity(line):
	"""
	Extract named entity from current line.

	Args:
		line (str): Current line

	Returns:
		str or None: Extracted named entity or None if no named entity is present.
	"""
	try:
		line_parts = line.split("\t")
		feature = line_parts[3]
		if feature != "O":
			return line_parts[2], line_parts[3]
	except IndexError:
		return None


if __name__ == '__main__':
	main()

#!/usr/bin/python # -*- coding: utf-8 -*- Convert the *DECOW14X* corpus into a plain text file. Is used as pre-processing step for the `word2vec <https://code.google.com/archive/p/word2vec/>`_ training. To make this this more feasible (decow is a **huge** corpus), python's :mod:`multiprocessing` is used, s.t. every part of the corpus in simultaneously processed. Afterwards, a bash command like ``cat`` can be used to merge into one single file. # STANDARD # PROJECT Main function. Uses command lines to start corpus processing. Convert the whole corpus into plain text.  	Args: 		decow_dir (str): Path to directory with decow corpus paths. 		out_dir (str): Path where plain text parts should be written to. 		log_path (str): Path where the log files should be written to. 		merge_nes (bool): Flag to indicate whether multi-word expression should be merged with underscores. 		log_interval (int): Interval to log current process state in seconds. # Split logging interval into hourse - minutes - seconds # Init logfile # Start processes Convert a corpus part into plain text without merging multiple word entries.  	Args: 		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath`` 					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval`` 					(*int*): Logging interval in seconds) Convert a corpus part into plain text and merging multiple word entries.  	Args: 		argstuple: Tuple of methods arguments (``inpath`` (*str*): Path to this processes' corpus part / ``dir_outpath`` 					(*str*): Path to this processes' output / ``log_path`` (*str*): Path to this processes' log / ``interval`` 					(*int*): Logging interval in seconds) # Extract possible named entity # If there was a named entity found, try to complete it if it's a multi-word expression Get the number of the current decow corpus part.  	Args: 		filename (str): Decow corpus part file name  	Returns: 		str: File number # Retrieve file number Extract named entity from current line.  	Args: 		line (str): Current line  	Returns: 		str or None: Extracted named entity or None if no named entity is present.