<s> import random_pmids <EOL> import requests <EOL> import os <EOL> import json <EOL> import re <EOL> import random <EOL> def get_nth_figshare_id ( url_template , year , n ) : <EOL> ( page_minus_one , offset ) = divmod ( n , <NUM_LIT:10> ) <EOL> url = url_template . format ( page = page_minus_one + <NUM_LIT:1> , year = year ) <EOL> print url <EOL> r = requests . get ( url , timeout = <NUM_LIT:10> ) <EOL> response = json . loads ( r . text ) <EOL> try : <EOL> items = response [ "<STR_LIT>" ] <EOL> item = items [ offset ] <EOL> doi = item [ "<STR_LIT>" ] <EOL> year_of_item = item [ "<STR_LIT>" ] [ - <NUM_LIT:4> : ] <EOL> except ( AttributeError ) : <EOL> print "<STR_LIT>" , item <EOL> doi = None <EOL> except KeyError : <EOL> print response <EOL> return get_nth_figshare_id ( url_template , year , n + <NUM_LIT:10> ) <EOL> doi = doi . replace ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> return ( { "<STR_LIT>" : doi , "<STR_LIT>" : year_of_item } ) <EOL> def get_random_figshare_dois ( year , sample_size , email , query , seed = None ) : <EOL> if year < <NUM_LIT> : <EOL> return [ ] <EOL> url_template = "<STR_LIT>" <EOL> url = url_template . format ( page = <NUM_LIT:1> , year = year ) <EOL> print url <EOL> r = requests . get ( url ) <EOL> initial_response = json . loads ( r . text ) <EOL> print initial_response <EOL> population_size = initial_response [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" % population_size <EOL> print "<STR_LIT>" % sample_size <EOL> if seed : <EOL> random . seed ( seed ) <EOL> print "<STR_LIT>" <EOL> if sample_size > population_size : <EOL> print "<STR_LIT>" <EOL> sample_size = population_size <EOL> random_indexes = random . sample ( range ( <NUM_LIT:1> , population_size ) , sample_size ) <EOL> figshare_ids = [ ] <EOL> for random_index in random_indexes : <EOL> figshare_dict = get_nth_figshare_id ( url_template , year , random_index ) <EOL> doi = figshare_dict [ "<STR_LIT>" ] <EOL> print "<STR_LIT>" + doi <EOL> figshare_ids . append ( doi ) <EOL> return figshare_ids <EOL> def get_random_dryad_dois ( year , sample_size , email , query , seed = None ) : <EOL> if year < <NUM_LIT> : <EOL> return [ ] <EOL> url = "<STR_LIT>" . format ( year = year ) <EOL> print url <EOL> try : <EOL> r = requests . get ( url , timeout = <NUM_LIT:10> ) <EOL> except requests . Timeout : <EOL> print "<STR_LIT>" <EOL> return [ ] <EOL> all_dois = re . findall ( "<STR_LIT>" , r . text ) <EOL> print all_dois [ <NUM_LIT:1> ] <EOL> print len ( all_dois ) <EOL> if seed : <EOL> random . seed ( seed ) <EOL> random_dois = random . sample ( all_dois , sample_size ) <EOL> return random_dois <EOL> def get_random_crossref_dois ( year , sample_size ) : <EOL> url = "<STR_LIT>" + str ( sample_size ) <EOL> try : <EOL> r = requests . get ( url , timeout = <NUM_LIT:10> ) <EOL> except requests . Timeout : <EOL> print "<STR_LIT>" <EOL> return [ ] <EOL> dois = json . loads ( r . text ) <EOL> return dois <EOL> def get_random_github_dict ( ) : <EOL> from totalimpact . providers import github <EOL> filename = "<STR_LIT>" <EOL> import collections <EOL> import subprocess <EOL> command = "<STR_LIT>" + filename <EOL> p = subprocess . Popen ( command , stdout = subprocess . PIPE , stderr = subprocess . PIPE , shell = True ) <EOL> out , error = p . communicate ( ) <EOL> num_repos = int ( out . strip ( ) . split ( "<STR_LIT:U+0020>" ) [ <NUM_LIT:0> ] ) <EOL> print num_repos <EOL> seed = <NUM_LIT> <EOL> random . seed ( seed ) <EOL> provider = github . Github ( ) <EOL> random_repos = collections . defaultdict ( list ) <EOL> i = <NUM_LIT:0> <EOL> while ( len ( random_repos [ "<STR_LIT>" ] ) < <NUM_LIT:100> ) : <EOL> i += <NUM_LIT:1> <EOL> line_number = random . randint ( <NUM_LIT:0> , num_repos ) <EOL> command = "<STR_LIT>" . format ( <EOL> line_number = line_number , filename = filename ) <EOL> p = subprocess . Popen ( command , stdout = subprocess . PIPE , stderr = subprocess . PIPE , shell = True ) <EOL> repo_id , error = p . communicate ( ) <EOL> print repo_id <EOL> repo_url = "<STR_LIT>" + repo_id . strip ( ) <EOL> biblio = provider . biblio ( [ ( "<STR_LIT:url>" , repo_url ) ] ) <EOL> try : <EOL> year = biblio [ "<STR_LIT>" ] [ <NUM_LIT:0> : <NUM_LIT:4> ] <EOL> except KeyError : <EOL> continue <EOL> random_repos [ year ] += [ repo_url ] <EOL> print "<STR_LIT>" . format ( number_sampled = i ) <EOL> for year in sorted ( random_repos . keys ( ) ) : <EOL> print "<STR_LIT>" . format ( year = year , num = len ( random_repos [ year ] ) ) <EOL> if i % <NUM_LIT:100> == <NUM_LIT:0> : <EOL> print random_repos <EOL> print random_repos <EOL> return random_repos <EOL> def get_random_github_ids ( year , sample_size , email , query , seed ) : <EOL> if year < <NUM_LIT> : <EOL> return [ ] <EOL> filename = "<STR_LIT>" <EOL> from random_github_repos import random_github_repos <EOL> if seed : <EOL> random . seed ( seed ) <EOL> random_ids = random . sample ( random_github_repos [ str ( year ) ] , sample_size ) <EOL> return random_ids <EOL> def make_collection ( namespace , nids , title , refset_metadata ) : <EOL> aliases = [ ( namespace , nid ) for nid in nids ] <EOL> url = api_url + "<STR_LIT>" + os . getenv ( "<STR_LIT>" ) <EOL> resp = requests . post ( <EOL> url , <EOL> data = json . dumps ( { <EOL> "<STR_LIT>" : aliases , <EOL> "<STR_LIT:title>" : title , <EOL> "<STR_LIT>" : refset_metadata <EOL> } ) , <EOL> headers = { '<STR_LIT>' : '<STR_LIT:application/json>' } <EOL> ) <EOL> collection_id = json . loads ( resp . text ) [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> print collection_id <EOL> return collection_id <EOL> def collection_metadata ( genre , refset_name , year , seed , sample_size ) : <EOL> refset_metadata = { <EOL> "<STR_LIT>" : genre , <EOL> "<STR_LIT:version>" : <NUM_LIT:0.1> , <EOL> "<STR_LIT:name>" : refset_name , <EOL> "<STR_LIT>" : year , <EOL> "<STR_LIT>" : seed , <EOL> "<STR_LIT>" : sample_size <EOL> } <EOL> return refset_metadata <EOL> def build_collections ( refset_name , get_random_ids_func , genre , namespace , query_template , title_template , year , sample_size , seed ) : <EOL> collection_ids = [ ] <EOL> print refset_name <EOL> if query_template : <EOL> query = query_template [ refset_name ] . format ( year = year ) <EOL> else : <EOL> query = None <EOL> ids = get_random_ids_func ( year , sample_size , email , query , seed ) <EOL> if ids : <EOL> print ids <EOL> refset_metadata = collection_metadata ( genre , refset_name , year , seed , sample_size ) <EOL> title = title_template . format ( ** refset_metadata ) <EOL> collection_id = make_collection ( namespace , ids , title , refset_metadata ) <EOL> print collection_id <EOL> collection_ids . append ( collection_id ) <EOL> return collection_ids <EOL> def build_reference_sets ( refset_name , query_templates , title_template , years , sample_size , seed ) : <EOL> collection_ids = [ ] <EOL> for year in years : <EOL> print year <EOL> if refset_name == "<STR_LIT>" : <EOL> collection_ids += build_collections ( refset_name , random_pmids . get_random_pmids , "<STR_LIT>" , "<STR_LIT>" , query_template , title_template , year , sample_size , seed ) <EOL> elif refset_name == "<STR_LIT>" : <EOL> collection_ids += build_collections ( refset_name , get_random_crossref_dois , "<STR_LIT>" , "<STR_LIT>" , None , title_template , year , sample_size , "<STR_LIT>" ) <EOL> elif refset_name == "<STR_LIT>" : <EOL> collection_ids += build_collections ( refset_name , get_random_dryad_dois , "<STR_LIT>" , "<STR_LIT>" , None , title_template , year , sample_size , seed ) <EOL> elif refset_name == "<STR_LIT>" : <EOL> collection_ids += build_collections ( refset_name , get_random_figshare_dois , "<STR_LIT>" , "<STR_LIT>" , None , title_template , year , sample_size , seed ) <EOL> elif refset_name == "<STR_LIT>" : <EOL> collection_ids += build_collections ( refset_name , get_random_github_ids , "<STR_LIT>" , "<STR_LIT:url>" , None , title_template , year , sample_size , seed ) <EOL> return ( collection_ids ) <EOL> query_templates = { <EOL> } <EOL> title_template = "<STR_LIT>" <EOL> email = "<STR_LIT>" <EOL> api_url = "<STR_LIT>" <EOL> sample_size = <NUM_LIT:100> <EOL> seed = <NUM_LIT> <EOL> years = range ( <NUM_LIT> , <NUM_LIT> ) <EOL> refset_name = "<STR_LIT>" <EOL> collection_ids = build_reference_sets ( refset_name , query_templates , title_template , years , sample_size , seed ) <EOL> for collection_id in collection_ids : <EOL> print collection_id </s>
