<s> """<STR_LIT>""" <EOL> import argparse <EOL> import collections <EOL> import logging <EOL> import re <EOL> import rethinkdb as r <EOL> import db . util <EOL> from db . github_repos import PluginGithubRepos <EOL> r_conn = db . util . r_conn <EOL> _GITHUB_REPO_URL_PATTERN = re . compile ( <EOL> r'<STR_LIT>' , re . IGNORECASE ) <EOL> def _normalize_github_url ( url ) : <EOL> """<STR_LIT>""" <EOL> url = re . sub ( r'<STR_LIT>' , '<STR_LIT>' , url ) <EOL> url = re . sub ( r'<STR_LIT>' , '<STR_LIT>' , url ) <EOL> return url . lower ( ) <EOL> def _is_github_url_allowed ( url ) : <EOL> """<STR_LIT>""" <EOL> parts = url . split ( '<STR_LIT:/>' ) <EOL> if len ( parts ) != <NUM_LIT:3> or len ( parts [ <NUM_LIT:2> ] ) < <NUM_LIT:1> or parts [ <NUM_LIT:2> ] == '<STR_LIT:\\>' : <EOL> return False <EOL> if parts [ <NUM_LIT:1> ] == '<STR_LIT>' : <EOL> return False <EOL> if parts [ <NUM_LIT:1> ] == '<STR_LIT>' : <EOL> return False <EOL> return True <EOL> def _extract_github_repo_urls ( text ) : <EOL> """<STR_LIT>""" <EOL> matches = _GITHUB_REPO_URL_PATTERN . findall ( text ) <EOL> normalized_matches = map ( _normalize_github_url , matches ) <EOL> return filter ( _is_github_url_allowed , normalized_matches ) <EOL> def get_repos_from_vimorg_descriptions ( ) : <EOL> """<STR_LIT>""" <EOL> print "<STR_LIT>" <EOL> repo_urls_dict = collections . defaultdict ( set ) <EOL> all_plugins = r . table ( '<STR_LIT>' ) . run ( r_conn ( ) ) <EOL> for plugin in all_plugins : <EOL> for field in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if field in plugin and plugin [ field ] : <EOL> repo_urls = set ( _extract_github_repo_urls ( plugin [ field ] ) ) <EOL> vimorg_id = plugin [ '<STR_LIT>' ] <EOL> if not vimorg_id : <EOL> continue <EOL> for repo_url in repo_urls : <EOL> repo_urls_dict [ repo_url ] . add ( vimorg_id ) <EOL> num_inserted = <NUM_LIT:0> <EOL> for repo_url , vimorg_ids in repo_urls_dict . iteritems ( ) : <EOL> _ , owner , repo_name = repo_url . split ( '<STR_LIT:/>' ) <EOL> inserted = PluginGithubRepos . upsert_with_owner_repo ( { <EOL> '<STR_LIT>' : owner , <EOL> '<STR_LIT>' : repo_name , <EOL> '<STR_LIT>' : list ( vimorg_ids ) , <EOL> } ) <EOL> num_inserted += int ( inserted ) <EOL> print "<STR_LIT>" % ( <EOL> len ( repo_urls_dict ) , num_inserted ) <EOL> def aggregate_repos_from_dotfiles ( ) : <EOL> """<STR_LIT>""" <EOL> users_counter = collections . Counter ( ) <EOL> repos_counter = collections . Counter ( ) <EOL> manager_counter = collections . Counter ( ) <EOL> managers = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> query = r . table ( '<STR_LIT>' ) . pluck ( managers . values ( ) ) <EOL> all_dotfiles = query . run ( r_conn ( ) ) <EOL> for dotfiles_repo in all_dotfiles : <EOL> for manager , field in managers . iteritems ( ) : <EOL> plugin_repos = dotfiles_repo [ field ] <EOL> users_counter [ manager ] += <NUM_LIT:1> if plugin_repos else <NUM_LIT:0> <EOL> manager_counter [ manager ] += len ( plugin_repos ) <EOL> for owner_repo in plugin_repos : <EOL> owner_repo = owner_repo . lower ( ) <EOL> repos_counter [ owner_repo ] += <NUM_LIT:1> <EOL> num_inserted = <NUM_LIT:0> <EOL> for owner_repo , num_users in repos_counter . iteritems ( ) : <EOL> owner , repo_name = owner_repo . split ( '<STR_LIT:/>' ) <EOL> repo = { <EOL> '<STR_LIT>' : owner , <EOL> '<STR_LIT>' : repo_name , <EOL> '<STR_LIT>' : num_users , <EOL> } <EOL> newly_inserted = PluginGithubRepos . upsert_with_owner_repo ( repo ) <EOL> num_inserted += int ( newly_inserted ) <EOL> most_used_repos = '<STR_LIT:\n>' . join ( map ( str , repos_counter . most_common ( <NUM_LIT:10> ) ) ) <EOL> print '<STR_LIT>' , most_used_repos <EOL> print '<STR_LIT>' , users_counter <EOL> print '<STR_LIT>' , manager_counter <EOL> print '<STR_LIT>' , query . count ( ) . run ( r_conn ( ) ) <EOL> print '<STR_LIT>' , num_inserted <EOL> print '<STR_LIT>' , len ( repos_counter ) <EOL> print '<STR_LIT>' , sum ( manager_counter . values ( ) ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> parser = argparse . ArgumentParser ( ) <EOL> extract_fns = { <EOL> "<STR_LIT>" : get_repos_from_vimorg_descriptions , <EOL> "<STR_LIT>" : aggregate_repos_from_dotfiles , <EOL> } <EOL> parser . add_argument ( "<STR_LIT>" , "<STR_LIT>" , choices = extract_fns . keys ( ) , <EOL> default = "<STR_LIT:all>" , help = "<STR_LIT>" ) <EOL> args = parser . parse_args ( ) <EOL> sources = extract_fns . keys ( ) if args . source == "<STR_LIT:all>" else [ args . source ] <EOL> for source in sources : <EOL> extract_fn = extract_fns [ source ] <EOL> try : <EOL> extract_fn ( ) <EOL> except Exception : <EOL> logging . exception ( "<STR_LIT>" % ( <EOL> extract_fn ) ) </s>
