# -*- coding: utf-8 -*-
"""Convert the Yelp Dataset Challenge dataset from json format to csv.
For more information on the Yelp Dataset Challenge please visit http://yelp.com/dataset_challenge
"""
import argparse
import codecs
import collections
import csv
import simplejson as json


def read_and_write_file(json_file_path, csv_file_path, column_names):
    """Read in the json dataset file and write it out to a csv file, given the column names."""
    with open(csv_file_path, 'wb+') as fout:
        csv_file = csv.writer(fout)
        csv_file.writerow(list(column_names))
        with open(json_file_path) as fin:
            for line in fin:
                print line
                line_contents = json.loads(line)
                csv_file.writerow(get_row(line_contents, column_names))

def get_superset_of_column_names_from_file(json_file_path):
    """Read in the json dataset file and return the superset of column names."""
    column_names = set()
    with open(json_file_path) as fin:
        for line in fin:
            print line
            line_contents = json.loads(line)
            column_names.update(
                    set(get_column_names(line_contents).keys())
                    )
    return column_names

def get_review_rating_from_file(json_file_path, limit=None):
    """Read in the json dataset file and return the reviews and ratings."""
    reviews = []
    num_processed = 0
    with open(json_file_path) as fin:
        for line in fin:
            num_processed += 1
            reviews.append(json.loads(line))
            if limit is not None and limit <= num_processed:
                break
    return reviews

def write_file(reviews, reviews_csv_file_path, ratings_csv_file_path):
    """Read in the json dataset file and write it out to a csv file, given the column names."""
    with open(reviews_csv_file_path, 'wb+') as reviews_out:
        with open(ratings_csv_file_path, 'wb+') as ratings_out:
            reviews_out.writelines([get_row(review, ['text'])[0]+'\n' for review in reviews])
            ratings_out.writelines([get_row(review, ['stars'])[0]+'\n' for review in reviews])


def get_column_names(line_contents, parent_key=''):
    """Return a list of flattened key names given a dict.
    Example:
        line_contents = {
            'a': {
                'b': 2,
                'c': 3,
                },
        }
        will return: ['a.b', 'a.c']
    These will be the column names for the eventual csv file.
    """
    column_names = []
    for k, v in line_contents.iteritems():
        column_name = "{0}.{1}".format(parent_key, k) if parent_key else k
        if isinstance(v, collections.MutableMapping):
            column_names.extend(
                    get_column_names(v, column_name).items()
                    )
        else:
            column_names.append((column_name, v))
    return dict(column_names)

def get_nested_value(d, key):
    """Return a dictionary item given a dictionary `d` and a flattened key from `get_column_names`.

    Example:
        d = {
            'a': {
                'b': 2,
                'c': 3,
                },
        }
        key = 'a.b'
        will return: 2

    """
    if '.' not in key:
        if key not in d:
            return None
        return d[key]
    base_key, sub_key = key.split('.', 1)
    if base_key not in d:
        return None
    sub_dict = d[base_key]
    return get_nested_value(sub_dict, sub_key)

def get_row(line_contents, column_names):
    """Return a csv compatible row given column names and a dict."""
    row = []
    for column_name in column_names:
        line_value = get_nested_value(
                        line_contents,
                        column_name,
                        )
        if isinstance(line_value, unicode):
            row.append('{0}'.format(line_value.encode('utf-8')))
        elif line_value is not None:
            if column_name == 'stars':
                line_value = (line_value - 1)* 0.25
            row.append('{0}'.format(line_value))
        else:
            row.append('')
    row = [s.replace('\n', '').replace('\r', '') for s in row]
    return row


def get_review_rating(json_file, limit=None):
    review_csv_file = '{0}_review.txt'.format(json_file.split('.json')[0])
    rating_csv_file = '{0}_rating.txt'.format(json_file.split('.json')[0])
    reviews = get_review_rating_from_file(json_file, limit)
    write_file(reviews, review_csv_file, rating_csv_file)


if __name__ == '__main__':
    """Convert a yelp dataset file from json to csv."""

    parser = argparse.ArgumentParser(
            description='Convert Yelp Dataset Challenge data from JSON format to CSV.',
            )

    parser.add_argument(
            'json_file',
            type=str,
            help='The json file to convert.',
            )

    parser.add_argument(
        'limit',
        type=int,
        help='Limit of reviews to process',
        default=None,
        nargs='?',
    )

    args = parser.parse_args()

    json_file = args.json_file
    limit = args.limit
    get_review_rating(json_file, limit)

    # read_and_write_file(json_file, csv_file, column_names)

# -*- coding: utf-8 -*-
"""Convert the Yelp Dataset Challenge dataset from json format to csv.
For more information on the Yelp Dataset Challenge please visit http://yelp.com/dataset_challenge
"""
import argparse
import codecs
import collections
import csv
import simplejson as json


def read_and_write_file(json_file_path, csv_file_path, column_names):
    """Read in the json dataset file and write it out to a csv file, given the column names."""
    with open(csv_file_path, 'wb+') as fout:
        csv_file = csv.writer(fout)
        csv_file.writerow(list(column_names))
        with open(json_file_path) as fin:
            for line in fin:
                print line
                line_contents = json.loads(line)
                csv_file.writerow(get_row(line_contents, column_names))

def get_superset_of_column_names_from_file(json_file_path):
    """Read in the json dataset file and return the superset of column names."""
    column_names = set()
    with open(json_file_path) as fin:
        for line in fin:
            print line
            line_contents = json.loads(line)
            column_names.update(
                    set(get_column_names(line_contents).keys())
                    )
    return column_names

def get_review_rating_from_file(json_file_path, limit=None):
    """Read in the json dataset file and return the reviews and ratings."""
    reviews = []
    num_processed = 0
    with open(json_file_path) as fin:
        for line in fin:
            num_processed += 1
            reviews.append(json.loads(line))
            if limit is not None and limit <= num_processed:
                break
    return reviews

def write_file(reviews, reviews_csv_file_path, ratings_csv_file_path):
    """Read in the json dataset file and write it out to a csv file, given the column names."""
    with open(reviews_csv_file_path, 'wb+') as reviews_out:
        with open(ratings_csv_file_path, 'wb+') as ratings_out:
            reviews_out.writelines([get_row(review, ['text'])[0]+'\n' for review in reviews])
            ratings_out.writelines([get_row(review, ['stars'])[0]+'\n' for review in reviews])


def get_column_names(line_contents, parent_key=''):
    """Return a list of flattened key names given a dict.
    Example:
        line_contents = {
            'a': {
                'b': 2,
                'c': 3,
                },
        }
        will return: ['a.b', 'a.c']
    These will be the column names for the eventual csv file.
    """
    column_names = []
    for k, v in line_contents.iteritems():
        column_name = "{0}.{1}".format(parent_key, k) if parent_key else k
        if isinstance(v, collections.MutableMapping):
            column_names.extend(
                    get_column_names(v, column_name).items()
                    )
        else:
            column_names.append((column_name, v))
    return dict(column_names)

def get_nested_value(d, key):
    """Return a dictionary item given a dictionary `d` and a flattened key from `get_column_names`.

    Example:
        d = {
            'a': {
                'b': 2,
                'c': 3,
                },
        }
        key = 'a.b'
        will return: 2

    """
    if '.' not in key:
        if key not in d:
            return None
        return d[key]
    base_key, sub_key = key.split('.', 1)
    if base_key not in d:
        return None
    sub_dict = d[base_key]
    return get_nested_value(sub_dict, sub_key)

def get_row(line_contents, column_names):
    """Return a csv compatible row given column names and a dict."""
    row = []
    for column_name in column_names:
        line_value = get_nested_value(
                        line_contents,
                        column_name,
                        )
        if isinstance(line_value, unicode):
            row.append('{0}'.format(line_value.encode('utf-8')))
        elif line_value is not None:
            if column_name == 'stars':
                line_value = (line_value - 1)* 0.25
            row.append('{0}'.format(line_value))
        else:
            row.append('')
    row = [s.replace('\n', '').replace('\r', '') for s in row]
    return row


def get_review_rating(json_file, limit=None):
    review_csv_file = '{0}_review.txt'.format(json_file.split('.json')[0])
    rating_csv_file = '{0}_rating.txt'.format(json_file.split('.json')[0])
    reviews = get_review_rating_from_file(json_file, limit)
    write_file(reviews, review_csv_file, rating_csv_file)


if __name__ == '__main__':
    """Convert a yelp dataset file from json to csv."""

    parser = argparse.ArgumentParser(
            description='Convert Yelp Dataset Challenge data from JSON format to CSV.',
            )

    parser.add_argument(
            'json_file',
            type=str,
            help='The json file to convert.',
            )

    parser.add_argument(
        'limit',
        type=int,
        help='Limit of reviews to process',
        default=None,
        nargs='?',
    )

    args = parser.parse_args()

    json_file = args.json_file
    limit = args.limit
    get_review_rating(json_file, limit)

    # read_and_write_file(json_file, csv_file, column_names)

# -*- coding: utf-8 -*- Convert the Yelp Dataset Challenge dataset from json format to csv. For more information on the Yelp Dataset Challenge please visit http://yelp.com/dataset_challenge Read in the json dataset file and write it out to a csv file, given the column names. Read in the json dataset file and return the superset of column names. Read in the json dataset file and return the reviews and ratings. Read in the json dataset file and write it out to a csv file, given the column names. Return a list of flattened key names given a dict.     Example:         line_contents = {             'a': {                 'b': 2,                 'c': 3,                 },         }         will return: ['a.b', 'a.c']     These will be the column names for the eventual csv file. Return a dictionary item given a dictionary `d` and a flattened key from `get_column_names`.      Example:         d = {             'a': {                 'b': 2,                 'c': 3,                 },         }         key = 'a.b'         will return: 2 Return a csv compatible row given column names and a dict. Convert a yelp dataset file from json to csv. # read_and_write_file(json_file, csv_file, column_names)