<s> """<STR_LIT>""" <EOL> __author__ = '<STR_LIT>' <EOL> __version__ = '<STR_LIT>' <EOL> from collections import Counter <EOL> import datetime <EOL> import csv <EOL> import cPickle <EOL> import functools <EOL> import pprint <EOL> import operator <EOL> import os <EOL> import re <EOL> import sqlite3 <EOL> import subprocess <EOL> import numpy as np <EOL> import matplotlib . pyplot as plt <EOL> minSubredditCommentsToCountAsBig = <NUM_LIT:1000> <EOL> fixedStartDate = <NUM_LIT> <EOL> days_to_go_back = <NUM_LIT> <EOL> languages = [ <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ ] ) , <EOL> ( '<STR_LIT>' , [ ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> ] <EOL> subreddit_has_alias = dict ( map ( lambda x : ( x [ <NUM_LIT:0> ] , True if x [ <NUM_LIT:1> ] else False ) , languages ) ) <EOL> tiobe_values = { <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> , <EOL> '<STR_LIT>' : <NUM_LIT> <EOL> } <EOL> subreddits = map ( lambda x : x [ <NUM_LIT:0> ] , languages ) <EOL> emotions = [ <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ] <EOL> positive_emotions = sorted ( map ( operator . itemgetter ( <NUM_LIT:0> ) , emotions ) ) <EOL> negative_emotions = sorted ( map ( operator . itemgetter ( <NUM_LIT:1> ) , emotions ) ) <EOL> internet_slang_words = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> opposite_words = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT:data>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> single_words = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:error>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:object>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:user>' , <EOL> '<STR_LIT>' ] <EOL> emoticons = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> def make_dir ( directory ) : <EOL> if not os . path . exists ( directory ) : <EOL> os . makedirs ( directory ) <EOL> def utf_to_ascii ( str ) : <EOL> return str . encode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> def replace_unwanted_chars ( str ) : <EOL> return str . replace ( '<STR_LIT:U+002C>' , '<STR_LIT:->' ) . replace ( '<STR_LIT:\r>' , '<STR_LIT:->' ) . replace ( '<STR_LIT:\n>' , '<STR_LIT:->' ) . replace ( '<STR_LIT::>' , '<STR_LIT:->' ) <EOL> def submission_meta ( submission ) : <EOL> return '<STR_LIT>' + str ( int ( submission . created_utc ) ) + '<STR_LIT>' + str ( submission . ups ) + '<STR_LIT>' + str ( submission . downs ) + '<STR_LIT>' + replace_unwanted_chars ( utf_to_ascii ( submission . title ) ) + '<STR_LIT>' + str ( submission . num_comments ) + '<STR_LIT>' + replace_unwanted_chars ( utf_to_ascii ( str ( submission . author ) ) ) + '<STR_LIT>' + replace_unwanted_chars ( utf_to_ascii ( submission . selftext ) ) <EOL> def comment_meta ( comment ) : <EOL> return '<STR_LIT>' + str ( int ( comment . created_utc ) ) + '<STR_LIT>' + str ( comment . ups ) + '<STR_LIT>' + str ( comment . downs ) + '<STR_LIT>' + replace_unwanted_chars ( utf_to_ascii ( str ( comment . author ) ) ) + '<STR_LIT>' + replace_unwanted_chars ( utf_to_ascii ( comment . body ) ) <EOL> def write_str_to_file ( path , str ) : <EOL> with open ( path , '<STR_LIT:w>' ) as text_file : <EOL> text_file . write ( utf_to_ascii ( str ) ) <EOL> def get_ups_and_downs ( ratio , post ) : <EOL> ups = int ( round ( ( ratio * post . score ) / ( <NUM_LIT:2> * ratio - <NUM_LIT:1> ) ) if ratio != <NUM_LIT:0.5> else round ( post . score / <NUM_LIT:2> ) ) <EOL> downs = ups - post . score <EOL> return ( ups , downs ) <EOL> def get_comments ( ) : <EOL> import praw <EOL> r = praw . Reddit ( '<STR_LIT>' ) <EOL> make_dir ( '<STR_LIT>' ) <EOL> for i , subreddit in enumerate ( subreddits ) : <EOL> with open ( '<STR_LIT>' + subreddit + '<STR_LIT>' ) as text_file : <EOL> ids = text_file . read ( ) . split ( '<STR_LIT:\n>' ) <EOL> ids = filter ( None , ids ) <EOL> for j , submission_id in enumerate ( ids ) : <EOL> try : <EOL> submission = r . get_submission ( submission_id = submission_id ) <EOL> ( submission . ups , submission . downs ) = get_ups_and_downs ( submission . upvote_ratio , submission ) <EOL> subm_dir_base = '<STR_LIT>' + subreddit <EOL> submission_dir = subm_dir_base + '<STR_LIT:/>' + submission_id <EOL> make_dir ( submission_dir ) <EOL> write_str_to_file ( submission_dir + '<STR_LIT>' , ( submission_meta ( submission ) ) ) <EOL> submission . replace_more_comments ( limit = None ) <EOL> flat_comments = list ( praw . helpers . flatten_tree ( submission . comments ) ) <EOL> print '<STR_LIT>' % ( submission_id , j + <NUM_LIT:1> , len ( ids ) , len ( flat_comments ) , i + <NUM_LIT:1> , <EOL> len ( subreddits ) , subreddit ) <EOL> for comment in flat_comments : <EOL> if not hasattr ( comment , '<STR_LIT:id>' ) or not hasattr ( comment , '<STR_LIT:body>' ) : <EOL> continue <EOL> comment_path = submission_dir + '<STR_LIT:/>' + comment . id + "<STR_LIT>" <EOL> write_str_to_file ( comment_path , comment_meta ( comment ) ) <EOL> except Exception as e : <EOL> print "<STR_LIT>" , str ( e ) <EOL> print e . __doc__ <EOL> print e . message <EOL> def get_today_start_as_unix_timestamp ( ) : <EOL> now = datetime . datetime . now ( ) <EOL> sec_since_epoch = ( now . date ( ) - datetime . date ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) ) . total_seconds ( ) <EOL> return int ( sec_since_epoch ) <EOL> def get_submission_ids ( ) : <EOL> startT = get_today_start_as_unix_timestamp ( ) <EOL> if fixedStartDate : <EOL> startT = fixedStartDate <EOL> bash_script_file_name = '<STR_LIT>' <EOL> with open ( bash_script_file_name , '<STR_LIT:w>' ) as bash_script : <EOL> bash_script . write ( '<STR_LIT>' ) <EOL> bash_script . write ( '<STR_LIT>' ) <EOL> bash_script . write ( '<STR_LIT>' ) <EOL> for subreddit in subreddits : <EOL> bash_script . write ( '<STR_LIT>' % subreddit ) <EOL> bash_script . write ( '<STR_LIT>' % subreddit ) <EOL> t = startT <EOL> day_cnt = <NUM_LIT:0> <EOL> while t > <NUM_LIT> : <EOL> day_cnt += <NUM_LIT:1> <EOL> if day_cnt > days_to_go_back : <EOL> break <EOL> newT = t - ( <NUM_LIT> * <NUM_LIT> * <NUM_LIT> ) <EOL> bash_script . write ( '<STR_LIT>' % ( subreddit , newT , t , subreddit , newT , t ) ) <EOL> t = newT <EOL> bash_script . write ( "<STR_LIT>" % ( subreddit , subreddit , subreddit ) ) <EOL> subprocess . call ( [ '<STR_LIT>' , bash_script_file_name ] ) <EOL> os . remove ( bash_script_file_name ) <EOL> def load_comment ( path ) : <EOL> lines = [ line . strip ( ) for line in open ( path ) ] <EOL> pairs = [ line . split ( "<STR_LIT::>" , <NUM_LIT:1> ) for line in lines ] <EOL> return dict ( pairs ) <EOL> def load_comments ( subreddit ) : <EOL> comments = { } <EOL> mainDir = "<STR_LIT>" + subreddit <EOL> make_dir ( mainDir ) <EOL> for submission_id in os . listdir ( mainDir ) : <EOL> path = os . path . join ( mainDir , submission_id ) <EOL> if os . path . isdir ( path ) : <EOL> subDirPath = path <EOL> for fileName in os . listdir ( subDirPath ) : <EOL> if fileName . endswith ( "<STR_LIT>" ) : <EOL> filePath = os . path . join ( subDirPath , fileName ) <EOL> comment_id = fileName [ : - <NUM_LIT:4> ] <EOL> comments [ comment_id ] = load_comment ( filePath ) <EOL> comments [ comment_id ] [ '<STR_LIT>' ] = submission_id <EOL> return comments <EOL> def pickle_comments ( ) : <EOL> allComments = { } <EOL> for subreddit , aliases in languages : <EOL> print subreddit <EOL> allComments [ subreddit ] = load_comments ( subreddit ) <EOL> make_dir ( '<STR_LIT>' ) <EOL> write_str_to_file ( "<STR_LIT>" , <EOL> cPickle . dumps ( allComments ) ) <EOL> def minus_pad ( s ) : <EOL> return '<STR_LIT:->' + s + '<STR_LIT:->' <EOL> def prepare_comment_body ( s ) : <EOL> return re . sub ( r'<STR_LIT>' , '<STR_LIT:->' , s ) <EOL> def comments_to_db ( ) : <EOL> all_comments = cPickle . load ( open ( "<STR_LIT>" , "<STR_LIT:rb>" ) ) <EOL> print '<STR_LIT>' , len ( all_comments ) <EOL> dbPath = '<STR_LIT>' <EOL> if os . path . exists ( dbPath ) : <EOL> os . remove ( dbPath ) <EOL> conn = sqlite3 . connect ( dbPath ) <EOL> c = conn . cursor ( ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> for subreddit , aliases in languages : <EOL> c . execute ( '<STR_LIT>' , ( subreddit , ) ) <EOL> for alias in aliases : <EOL> c . execute ( '<STR_LIT>' , ( alias , subreddit ) ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> for subreddit , comments in all_comments . iteritems ( ) : <EOL> print subreddit <EOL> for comment_id , comment in comments . iteritems ( ) : <EOL> command = '<STR_LIT>' <EOL> c . execute ( command , ( comment_id , <EOL> subreddit , <EOL> comment [ '<STR_LIT>' ] , <EOL> comment [ '<STR_LIT>' ] , <EOL> comment [ '<STR_LIT>' ] , <EOL> comment [ '<STR_LIT>' ] , <EOL> comment [ '<STR_LIT>' ] , <EOL> minus_pad ( prepare_comment_body ( comment [ '<STR_LIT:body>' ] ) ) ) ) <EOL> conn . commit ( ) <EOL> conn . close ( ) <EOL> def get_mention_count ( c , mentioner , mentionee ) : <EOL> command = "<STR_LIT>" <EOL> c . execute ( command , ( mentioner , mentionee ) ) <EOL> return int ( c . fetchone ( ) [ <NUM_LIT:0> ] ) <EOL> def cache_mutual_mentions ( ) : <EOL> dbPath = '<STR_LIT>' <EOL> conn = sqlite3 . connect ( dbPath ) <EOL> c = conn . cursor ( ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> for mentionee , _ in languages : <EOL> print mentionee , <EOL> for mentioner , _ in languages : <EOL> result = get_mention_count ( c , mentioner , mentionee ) <EOL> print result , <EOL> c . execute ( '<STR_LIT>' , <EOL> ( mentionee , mentioner , result ) ) <EOL> print <EOL> conn . commit ( ) <EOL> conn . close ( ) <EOL> def cache_subreddit_comment_counts ( ) : <EOL> dbPath = '<STR_LIT>' <EOL> conn = sqlite3 . connect ( dbPath ) <EOL> c = conn . cursor ( ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> c . execute ( '''<STR_LIT>''' ) <EOL> for subreddit in subreddits : <EOL> command = "<STR_LIT>" <EOL> c . execute ( command , ( subreddit , ) ) <EOL> result = int ( c . fetchone ( ) [ <NUM_LIT:0> ] ) <EOL> print subreddit , result <EOL> c . execute ( '<STR_LIT>' , <EOL> ( subreddit , result ) ) <EOL> conn . commit ( ) <EOL> conn . close ( ) <EOL> def print_table ( header , table ) : <EOL> result = '<STR_LIT:U+002C>' . join ( header ) + "<STR_LIT:\n>" <EOL> for row in table : <EOL> for cell in row : <EOL> result += str ( cell ) + "<STR_LIT:U+002C>" <EOL> result = result [ : - <NUM_LIT:1> ] <EOL> result += "<STR_LIT:\n>" <EOL> return result <EOL> def relative_word_count ( c , subreddit , word ) : <EOL> command = "<STR_LIT>" <EOL> c . execute ( command , ( subreddit , subreddit , '<STR_LIT>' + word + '<STR_LIT>' ) ) <EOL> res = c . fetchone ( ) [ <NUM_LIT:0> ] <EOL> if not res : <EOL> return <NUM_LIT:0> <EOL> return int ( res ) <EOL> def get_subreddit_comment_count ( c , subreddit ) : <EOL> command = "<STR_LIT>" <EOL> c . execute ( command , ( subreddit , ) ) <EOL> return int ( c . fetchone ( ) [ <NUM_LIT:0> ] ) <EOL> def show_word_table ( c , subreddits , words ) : <EOL> print words <EOL> sanitized_words = map ( prepare_comment_body , words ) <EOL> sanitized_words = filter ( lambda x : x , sanitized_words ) <EOL> result = '<STR_LIT:U+002C>' . join ( [ "<STR_LIT>" ] + sanitized_words + [ "<STR_LIT>" ] ) <EOL> result += "<STR_LIT:\n>" <EOL> for subreddit in subreddits : <EOL> print subreddit , <EOL> cnt_sum = <NUM_LIT:0> <EOL> result += subreddit + "<STR_LIT:U+002C>" <EOL> for word in sanitized_words : <EOL> res = relative_word_count ( c , subreddit , word ) <EOL> cnt_sum += res <EOL> result += str ( res ) + "<STR_LIT:U+002C>" <EOL> result += str ( cnt_sum ) + "<STR_LIT:\n>" <EOL> return result <EOL> def transpose_table ( mat ) : <EOL> return zip ( * mat ) <EOL> def save_mutual_mentions ( c ) : <EOL> colors = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> big_languages = filter ( lambda ( s , _ ) : isSubredditBig ( c , s ) , languages ) <EOL> big_languages = filter ( lambda ( _ , a ) : a , big_languages ) <EOL> namecolorlist = "<STR_LIT>" <EOL> matrix = '<STR_LIT:[>' <EOL> result = '<STR_LIT:U+002C>' . join ( map ( operator . itemgetter ( <NUM_LIT:0> ) , [ ( "<STR_LIT>" , [ ] ) ] + big_languages ) ) <EOL> result += "<STR_LIT:\n>" <EOL> colorCnt = <NUM_LIT:0> <EOL> for mentionee , mentinee_aliases in big_languages : <EOL> if not mentinee_aliases : <EOL> continue <EOL> namecolorlist += mentionee + "<STR_LIT:U+002C>" + colors [ colorCnt ] + "<STR_LIT:\n>" <EOL> colorCnt += <NUM_LIT:1> <EOL> colorCnt = colorCnt % len ( colors ) <EOL> matrix += '<STR_LIT>' <EOL> result += mentionee + "<STR_LIT:U+002C>" <EOL> for mentioner , mentioner_aliases in big_languages : <EOL> if not mentioner_aliases : <EOL> continue <EOL> command = "<STR_LIT>" <EOL> c . execute ( command , ( mentioner , mentionee ) ) <EOL> intres = int ( c . fetchone ( ) [ <NUM_LIT:0> ] ) <EOL> if mentioner is mentionee : <EOL> intres = <NUM_LIT:0> <EOL> result += str ( intres ) + "<STR_LIT:U+002C>" <EOL> matrix += str ( intres ) + "<STR_LIT:U+002C>" <EOL> result += "<STR_LIT:\n>" <EOL> matrix = matrix [ : - <NUM_LIT:1> ] <EOL> matrix += '<STR_LIT>' <EOL> matrix = matrix [ : - <NUM_LIT:1> ] <EOL> matrix += "<STR_LIT>" <EOL> make_dir ( '<STR_LIT>' ) <EOL> write_str_to_file ( "<STR_LIT>" , result ) ; <EOL> write_str_to_file ( "<STR_LIT>" , matrix ) ; <EOL> write_str_to_file ( "<STR_LIT>" , namecolorlist ) ; <EOL> def who_by_whom ( c ) : <EOL> print '<STR_LIT>' <EOL> command = '<STR_LIT>' <EOL> c . execute ( command ) <EOL> write_str_to_file ( "<STR_LIT>" , print_table ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , c . fetchall ( ) ) ) <EOL> def who_himself ( c ) : <EOL> print '<STR_LIT>' <EOL> command = '<STR_LIT>' <EOL> c . execute ( command ) <EOL> write_str_to_file ( "<STR_LIT>" , print_table ( [ '<STR_LIT>' , '<STR_LIT>' ] , c . fetchall ( ) ) ) <EOL> def who_by_others ( c ) : <EOL> print '<STR_LIT>' <EOL> command = '<STR_LIT>' <EOL> c . execute ( command ) <EOL> write_str_to_file ( "<STR_LIT>" , print_table ( [ '<STR_LIT>' , '<STR_LIT>' ] , c . fetchall ( ) ) ) <EOL> def count_word_mentions ( c ) : <EOL> big_subreddits = filter ( functools . partial ( isSubredditBig , c ) , subreddits ) <EOL> print big_subreddits <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , big_subreddits , positive_emotions ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , big_subreddits , negative_emotions ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , big_subreddits , internet_slang_words ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , big_subreddits , single_words + opposite_words ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , subreddits , positive_emotions ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , subreddits , negative_emotions ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , subreddits , internet_slang_words ) ) <EOL> write_str_to_file ( '<STR_LIT>' , show_word_table ( c , subreddits , single_words + opposite_words ) ) <EOL> def isSubredditBig ( c , subreddit ) : <EOL> return get_subreddit_comment_count ( c , subreddit ) >= minSubredditCommentsToCountAsBig <EOL> def cache_db_results ( ) : <EOL> cache_subreddit_comment_counts ( ) <EOL> cache_mutual_mentions ( ) <EOL> def analyse_comments ( ) : <EOL> dbPath = '<STR_LIT>' <EOL> conn = sqlite3 . connect ( dbPath ) <EOL> c = conn . cursor ( ) <EOL> who_by_whom ( c ) <EOL> who_himself ( c ) <EOL> who_by_others ( c ) <EOL> count_word_mentions ( c ) <EOL> save_mutual_mentions ( c ) <EOL> conn . close ( ) <EOL> def draw_word_mentions ( name , columns , colors , sorted_by_sum , filename , div_col = None ) : <EOL> reader = csv . DictReader ( open ( '<STR_LIT>' + name + '<STR_LIT>' ) ) <EOL> output = [ ] <EOL> for row in reader : <EOL> d = { } <EOL> for col in columns + [ '<STR_LIT>' ] : <EOL> d [ col ] = row [ col ] <EOL> if div_col : <EOL> d [ div_col ] = row [ div_col ] <EOL> output . append ( d ) <EOL> def div_row ( row , divisor ) : <EOL> return { k : int ( <NUM_LIT> * float ( x ) / float ( divisor ) ) if k != '<STR_LIT>' else x for k , x in row . items ( ) } <EOL> if div_col : <EOL> output = [ div_row ( row , row [ div_col ] ) for row in output ] <EOL> def dict_sum ( d ) : <EOL> c = d . copy ( ) ; <EOL> c . pop ( '<STR_LIT>' ) <EOL> return sum ( map ( int , c . values ( ) ) ) <EOL> if sorted_by_sum : <EOL> output . sort ( key = lambda x : dict_sum ( x ) , reverse = True ) <EOL> else : <EOL> output . sort ( key = lambda x : x [ '<STR_LIT>' ] , reverse = True ) <EOL> subreddits = [ ] <EOL> dataset = [ ] <EOL> for row in output : <EOL> subreddit = row [ '<STR_LIT>' ] <EOL> subreddits . append ( subreddit ) <EOL> dataset_row = { } <EOL> counts = [ ] <EOL> for column in columns : <EOL> dataset_row [ column ] = int ( row [ column ] ) <EOL> dataset . append ( dataset_row ) <EOL> data_orders = [ columns ] * len ( subreddits ) <EOL> if not dataset : <EOL> return <EOL> names = sorted ( dataset [ <NUM_LIT:0> ] . keys ( ) ) <EOL> values = np . array ( [ [ data [ name ] for name in order ] for data , order in zip ( dataset , data_orders ) ] ) <EOL> lefts = np . insert ( np . cumsum ( values , axis = <NUM_LIT:1> ) , <NUM_LIT:0> , <NUM_LIT:0> , axis = <NUM_LIT:1> ) [ : , : - <NUM_LIT:1> ] <EOL> orders = np . array ( data_orders ) <EOL> bottoms = np . arange ( len ( data_orders ) ) <EOL> for name , color in zip ( names , colors ) : <EOL> idx = np . where ( orders == name ) <EOL> value = values [ idx ] <EOL> left = lefts [ idx ] <EOL> plt . bar ( left = left , height = <NUM_LIT> , width = value , bottom = bottoms , <EOL> color = color , orientation = "<STR_LIT>" , label = name ) <EOL> plt . yticks ( bottoms + <NUM_LIT> , subreddits ) <EOL> plt . xlabel ( '<STR_LIT>' ) <EOL> plt . legend ( loc = "<STR_LIT>" , bbox_to_anchor = ( <NUM_LIT:1.0> , <NUM_LIT> ) ) <EOL> plt . savefig ( '<STR_LIT>' + filename + '<STR_LIT>' , bbox_inches = '<STR_LIT>' ) <EOL> plt . close ( ) <EOL> def draw_who_by_others ( ) : <EOL> table = [ ] <EOL> with open ( '<STR_LIT>' , '<STR_LIT:rb>' ) as csvfile : <EOL> reader = csv . reader ( csvfile , delimiter = '<STR_LIT:U+002C>' ) <EOL> next ( reader , None ) <EOL> for row in reader : <EOL> if not row : <EOL> continue <EOL> lang = row [ <NUM_LIT:0> ] <EOL> mention_val = float ( row [ <NUM_LIT:1> ] ) <EOL> if not lang in tiobe_values : <EOL> continue <EOL> tiobe_val = tiobe_values [ lang ] <EOL> if not tiobe_val : <EOL> continue <EOL> if not subreddit_has_alias [ lang ] : <EOL> continue <EOL> table . append ( ( lang , mention_val / tiobe_val ) ) <EOL> table . sort ( key = operator . itemgetter ( <NUM_LIT:0> ) , reverse = True ) <EOL> langs , vals = zip ( * table ) <EOL> y_pos = np . arange ( len ( langs ) ) <EOL> plt . barh ( y_pos , vals , align = '<STR_LIT>' , alpha = <NUM_LIT> ) <EOL> plt . yticks ( y_pos , langs ) <EOL> plt . title ( '<STR_LIT>' ) <EOL> plt . savefig ( '<STR_LIT>' , bbox_inches = '<STR_LIT>' ) <EOL> plt . close ( ) <EOL> def draw_graphs ( ) : <EOL> make_dir ( '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' ) <EOL> draw_who_by_others ( ) <EOL> def grep_irc ( ) : <EOL> subprocess . call ( [ '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> def draw_irc ( ) : <EOL> make_dir ( '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> draw_word_mentions ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> True , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> def main ( ) : <EOL> get_submission_ids ( ) <EOL> get_comments ( ) <EOL> pickle_comments ( ) <EOL> comments_to_db ( ) <EOL> cache_db_results ( ) <EOL> analyse_comments ( ) <EOL> grep_irc ( ) <EOL> draw_irc ( ) <EOL> draw_graphs ( ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
