<s> from datetime import datetime <EOL> import json <EOL> import re <EOL> import time <EOL> import urllib <EOL> import urlparse <EOL> from bs4 import BeautifulSoup , SoupStrainer <EOL> import feedparser <EOL> feedparser . _FeedParserMixin . namespaces [ <EOL> '<STR_LIT>' ] = '<STR_LIT>' <EOL> import requests <EOL> from vidscraper . exceptions import UnhandledVideo , UnhandledFeed <EOL> from vidscraper . suites import BaseSuite , registry <EOL> from vidscraper . utils . feedparser import struct_time_to_datetime <EOL> from vidscraper . videos import ( BaseFeed , BaseSearch , VideoLoader , <EOL> OEmbedLoaderMixin , VideoFile ) <EOL> class PathMixin ( object ) : <EOL> short_path_re = re . compile ( r"<STR_LIT>" ) <EOL> def get_url_data ( self , url ) : <EOL> parsed = urlparse . urlsplit ( url ) <EOL> if parsed . scheme in ( '<STR_LIT:http>' , '<STR_LIT>' ) : <EOL> if ( parsed . netloc in ( '<STR_LIT>' , '<STR_LIT>' ) and <EOL> parsed . path in ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> qsd = urlparse . parse_qs ( parsed . query ) <EOL> try : <EOL> return { <EOL> '<STR_LIT>' : qsd [ '<STR_LIT:v>' ] [ <NUM_LIT:0> ] <EOL> } <EOL> except ( KeyError , IndexError ) : <EOL> pass <EOL> elif parsed . netloc == '<STR_LIT>' : <EOL> match = self . short_path_re . match ( parsed . path ) <EOL> if match : <EOL> return match . groupdict ( ) <EOL> raise UnhandledVideo ( url ) <EOL> class ApiMixin ( object ) : <EOL> def get_headers ( self ) : <EOL> headers = super ( ApiMixin , self ) . get_headers ( ) <EOL> if '<STR_LIT>' in self . api_keys : <EOL> headers [ '<STR_LIT>' ] = "<STR_LIT>" + self . api_keys [ '<STR_LIT>' ] <EOL> return headers <EOL> def get_video_data ( self , item ) : <EOL> username = item [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] . rsplit ( '<STR_LIT:/>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if '<STR_LIT>' in item : <EOL> date_key = '<STR_LIT>' <EOL> else : <EOL> date_key = '<STR_LIT>' <EOL> media_group = item [ '<STR_LIT>' ] <EOL> description = media_group [ '<STR_LIT>' ] <EOL> if description [ '<STR_LIT:type>' ] != '<STR_LIT>' : <EOL> soup = BeautifulSoup ( description [ '<STR_LIT>' ] ) . findAll ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> description = unicode ( soup . string ) <EOL> else : <EOL> description = description [ '<STR_LIT>' ] <EOL> thumbnail_url = None <EOL> for thumbnail in media_group [ '<STR_LIT>' ] : <EOL> if thumbnail_url is None and thumbnail [ '<STR_LIT>' ] == '<STR_LIT:default>' : <EOL> thumbnail_url = thumbnail [ '<STR_LIT:url>' ] <EOL> if thumbnail [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> thumbnail_url = thumbnail [ '<STR_LIT:url>' ] <EOL> break <EOL> if '<STR_LIT>' in media_group [ '<STR_LIT>' ] : <EOL> tags = media_group [ '<STR_LIT>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT:U+002CU+0020>' ) <EOL> else : <EOL> tags = [ ] <EOL> tags . extend ( <EOL> [ cat [ '<STR_LIT>' ] for cat in media_group [ '<STR_LIT>' ] ] ) <EOL> data = { <EOL> '<STR_LIT>' : item [ '<STR_LIT>' ] [ <NUM_LIT:0> ] [ '<STR_LIT>' ] . split ( '<STR_LIT:&>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] , <EOL> '<STR_LIT:title>' : item [ '<STR_LIT:title>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT:description>' : description . replace ( '<STR_LIT:\r>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : thumbnail_url , <EOL> '<STR_LIT>' : datetime . strptime ( item [ date_key ] [ '<STR_LIT>' ] , <EOL> "<STR_LIT>" ) , <EOL> '<STR_LIT>' : tags , <EOL> '<STR_LIT:user>' : username , <EOL> '<STR_LIT>' : u'<STR_LIT>' . format ( username ) , <EOL> '<STR_LIT>' : '<STR_LIT>' . format ( <EOL> item [ '<STR_LIT:id>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT::>' ) [ - <NUM_LIT:1> ] ) , <EOL> '<STR_LIT>' : media_group [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : media_group [ '<STR_LIT>' ] [ '<STR_LIT:url>' ] <EOL> } <EOL> return data <EOL> class ApiLoader ( ApiMixin , PathMixin , VideoLoader ) : <EOL> fields = set ( ( '<STR_LIT>' , '<STR_LIT:title>' , '<STR_LIT:description>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:user>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> url_format = u"<STR_LIT>" <EOL> def get_video_data ( self , response ) : <EOL> if response . status_code in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> return { '<STR_LIT>' : False } <EOL> parsed = json . loads ( response . text ) <EOL> entry = parsed [ '<STR_LIT>' ] <EOL> return ApiMixin . get_video_data ( self , entry ) <EOL> class VideoInfoLoader ( PathMixin , VideoLoader ) : <EOL> fields = set ( ( '<STR_LIT:title>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> formats = ( <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT:5>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ( '<STR_LIT:100>' , u'<STR_LIT>' , <NUM_LIT> , <NUM_LIT> ) , <EOL> ) <EOL> url_format = u"<STR_LIT>" <EOL> def get_video_data ( self , response ) : <EOL> if response . status_code == <NUM_LIT> : <EOL> return { } <EOL> params = urlparse . parse_qs ( response . text . encode ( '<STR_LIT:utf-8>' ) ) <EOL> if params [ '<STR_LIT:status>' ] [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> if params [ '<STR_LIT>' ] [ <NUM_LIT:0> ] == '<STR_LIT>' : <EOL> return { '<STR_LIT>' : False } <EOL> return { } <EOL> data = { <EOL> '<STR_LIT:title>' : params [ '<STR_LIT:title>' ] [ <NUM_LIT:0> ] . decode ( '<STR_LIT:utf8>' ) , <EOL> '<STR_LIT>' : params [ '<STR_LIT>' ] [ <NUM_LIT:0> ] , <EOL> } <EOL> if '<STR_LIT>' in params : <EOL> data [ '<STR_LIT>' ] = params [ '<STR_LIT>' ] [ <NUM_LIT:0> ] . decode ( '<STR_LIT:utf8>' ) . split ( '<STR_LIT:U+002C>' ) <EOL> if data [ '<STR_LIT>' ] . endswith ( '<STR_LIT>' ) : <EOL> data [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] . replace ( <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> url_querystrings = params [ "<STR_LIT>" ] [ <NUM_LIT:0> ] . split ( "<STR_LIT:U+002C>" ) <EOL> url_data = [ urlparse . parse_qs ( qs ) for qs in url_querystrings ] <EOL> url_data_map = dict ( ( ud [ '<STR_LIT>' ] [ <NUM_LIT:0> ] , ud ) for ud in url_data if '<STR_LIT>' in ud ) <EOL> data [ '<STR_LIT>' ] = [ ] <EOL> for code , mime_type , width , height in self . formats : <EOL> if code in url_data_map : <EOL> file_data = url_data_map [ code ] <EOL> parsed_file_url = urlparse . urlsplit ( file_data [ '<STR_LIT:url>' ] [ <NUM_LIT:0> ] ) <EOL> parsed_file_url_qs = dict ( urlparse . parse_qsl ( parsed_file_url . query ) ) <EOL> expires = struct_time_to_datetime ( time . gmtime ( int ( parsed_file_url_qs [ '<STR_LIT>' ] ) ) ) <EOL> parsed_file_url_qs [ '<STR_LIT>' ] = file_data [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> url = urlparse . urlunsplit ( parsed_file_url [ : <NUM_LIT:3> ] + <EOL> ( urllib . urlencode ( parsed_file_url_qs ) , ) + <EOL> parsed_file_url [ <NUM_LIT:4> : ] ) <EOL> data [ '<STR_LIT>' ] . append ( VideoFile ( url = url , <EOL> expires = expires , <EOL> mime_type = mime_type , <EOL> width = width , <EOL> height = height ) ) <EOL> return data <EOL> class OEmbedLoader ( OEmbedLoaderMixin , PathMixin , VideoLoader ) : <EOL> endpoint = u"<STR_LIT>" <EOL> url_format = u"<STR_LIT>" <EOL> def get_video_data ( self , response ) : <EOL> if response . status_code in ( requests . codes . unauthorized , <EOL> requests . codes . forbidden ) : <EOL> return { '<STR_LIT>' : False } <EOL> if response . status_code == <NUM_LIT> : <EOL> return { } <EOL> return super ( OEmbedLoader , self ) . get_video_data ( response ) <EOL> class Feed ( ApiMixin , BaseFeed ) : <EOL> per_page = <NUM_LIT:50> <EOL> page_url_format = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> path_re = re . compile ( r'<STR_LIT>' ) <EOL> old_path_re = re . compile ( r'<STR_LIT>' ) <EOL> gdata_re = re . compile ( r'<STR_LIT>' ) <EOL> invalid_usernames = set ( ( '<STR_LIT:user>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def get_url_data ( self , url ) : <EOL> parsed_url = urlparse . urlsplit ( url ) <EOL> if parsed_url . scheme in ( '<STR_LIT:http>' , '<STR_LIT>' ) : <EOL> if parsed_url . netloc in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> match = self . path_re . match ( parsed_url . path ) <EOL> if match : <EOL> groupdict = match . groupdict ( ) <EOL> if groupdict [ '<STR_LIT:username>' ] not in self . invalid_usernames : <EOL> if groupdict [ '<STR_LIT:user>' ] is None : <EOL> response = requests . get ( url ) <EOL> if response . status_code == <NUM_LIT:200> : <EOL> strainer = SoupStrainer ( '<STR_LIT>' , <EOL> rel = '<STR_LIT>' ) <EOL> soup = BeautifulSoup ( response . content , <EOL> parse_only = strainer ) <EOL> if soup . link is not None : <EOL> return self . get_url_data ( soup . link [ '<STR_LIT>' ] ) <EOL> else : <EOL> return { '<STR_LIT:username>' : groupdict [ '<STR_LIT:username>' ] } <EOL> match = self . old_path_re . match ( parsed_url . path ) <EOL> if match : <EOL> parsed_qs = urlparse . parse_qs ( parsed_url . query ) <EOL> if '<STR_LIT:user>' in parsed_qs : <EOL> username = parsed_qs [ '<STR_LIT:user>' ] [ <NUM_LIT:0> ] <EOL> if username not in self . invalid_usernames : <EOL> return { '<STR_LIT:username>' : username } <EOL> elif parsed_url . netloc == '<STR_LIT>' : <EOL> match = self . gdata_re . match ( parsed_url . path ) <EOL> if match : <EOL> return match . groupdict ( ) <EOL> raise UnhandledFeed ( url ) <EOL> def get_response_items ( self , response ) : <EOL> return response . json ( ) [ '<STR_LIT>' ] . get ( '<STR_LIT>' , [ ] ) <EOL> def data_from_response ( self , response ) : <EOL> feed = response . json ( ) [ '<STR_LIT>' ] <EOL> for l in feed [ '<STR_LIT>' ] : <EOL> if l [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> link = l [ '<STR_LIT>' ] <EOL> break <EOL> else : <EOL> link = None <EOL> return { <EOL> '<STR_LIT>' : feed [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT:title>' : feed [ '<STR_LIT:title>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : link , <EOL> '<STR_LIT>' : feed [ '<STR_LIT:id>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : response . headers . get ( '<STR_LIT>' ) or feed [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : feed [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> } <EOL> class Search ( ApiMixin , BaseSearch ) : <EOL> per_page = <NUM_LIT:50> <EOL> page_url_format = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> order_by_map = { <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> def get_response_items ( self , response ) : <EOL> if response . status_code == <NUM_LIT> : <EOL> return [ ] <EOL> return response . json ( ) [ '<STR_LIT>' ] . get ( '<STR_LIT>' , [ ] ) <EOL> def data_from_response ( self , response ) : <EOL> if response . status_code == <NUM_LIT> : <EOL> return { } <EOL> feed = response . json ( ) [ '<STR_LIT>' ] <EOL> return { <EOL> '<STR_LIT>' : feed [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> } <EOL> class Suite ( BaseSuite ) : <EOL> loader_classes = ( OEmbedLoader , ApiLoader , <EOL> VideoInfoLoader ) <EOL> feed_class = Feed <EOL> search_class = Search <EOL> registry . register ( Suite ) </s>
