<s> import bs4 <EOL> import ClientNetworking <EOL> import collections <EOL> import httplib <EOL> import HydrusConstants as HC <EOL> import HydrusExceptions <EOL> import HydrusPaths <EOL> import HydrusSerialisable <EOL> import HydrusThreading <EOL> import json <EOL> import lxml <EOL> import os <EOL> import pafy <EOL> import re <EOL> import requests <EOL> import sys <EOL> import threading <EOL> import time <EOL> import traceback <EOL> import urllib <EOL> import urlparse <EOL> import wx <EOL> import HydrusTags <EOL> import HydrusData <EOL> import ClientConstants as CC <EOL> import HydrusGlobals <EOL> URL_EXTRA_INFO = { } <EOL> URL_EXTRA_INFO_LOCK = threading . Lock ( ) <EOL> def GetExtraURLInfo ( url ) : <EOL> with URL_EXTRA_INFO_LOCK : <EOL> if url in URL_EXTRA_INFO : <EOL> return URL_EXTRA_INFO [ url ] <EOL> else : <EOL> return None <EOL> def GetGalleryStreamIdentifiers ( gallery_identifier ) : <EOL> site_type = gallery_identifier . GetSiteType ( ) <EOL> if site_type == HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST : <EOL> gallery_stream_identifiers = [ GalleryIdentifier ( HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST_PICTURES ) , GalleryIdentifier ( HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST_SCRAPS ) ] <EOL> elif site_type == HC . SITE_TYPE_NEWGROUNDS : <EOL> gallery_stream_identifiers = [ GalleryIdentifier ( HC . SITE_TYPE_NEWGROUNDS_GAMES ) , GalleryIdentifier ( HC . SITE_TYPE_NEWGROUNDS_MOVIES ) ] <EOL> else : <EOL> gallery_stream_identifiers = [ gallery_identifier ] <EOL> return gallery_stream_identifiers <EOL> def SetExtraURLInfo ( url , info ) : <EOL> with URL_EXTRA_INFO_LOCK : <EOL> URL_EXTRA_INFO [ url ] = info <EOL> def GetGallery ( gallery_identifier ) : <EOL> site_type = gallery_identifier . GetSiteType ( ) <EOL> if site_type == HC . SITE_TYPE_BOORU : <EOL> booru_name = gallery_identifier . GetAdditionalInfo ( ) <EOL> return GalleryBooru ( booru_name ) <EOL> elif site_type == HC . SITE_TYPE_DEVIANT_ART : <EOL> return GalleryDeviantArt ( ) <EOL> elif site_type == HC . SITE_TYPE_GIPHY : <EOL> return GalleryGiphy ( ) <EOL> elif site_type in ( HC . SITE_TYPE_HENTAI_FOUNDRY , HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST ) : <EOL> return GalleryHentaiFoundry ( ) <EOL> elif site_type == HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST_PICTURES : <EOL> return GalleryHentaiFoundryArtistPictures ( ) <EOL> elif site_type == HC . SITE_TYPE_HENTAI_FOUNDRY_ARTIST_SCRAPS : <EOL> return GalleryHentaiFoundryArtistScraps ( ) <EOL> elif site_type == HC . SITE_TYPE_HENTAI_FOUNDRY_TAGS : <EOL> return GalleryHentaiFoundryTags ( ) <EOL> elif site_type == HC . SITE_TYPE_NEWGROUNDS : <EOL> return GalleryNewgrounds ( ) <EOL> elif site_type == HC . SITE_TYPE_NEWGROUNDS_GAMES : <EOL> return GalleryNewgroundsGames ( ) <EOL> elif site_type == HC . SITE_TYPE_NEWGROUNDS_MOVIES : <EOL> return GalleryNewgroundsMovies ( ) <EOL> elif site_type == HC . SITE_TYPE_PIXIV : <EOL> return GalleryPixiv ( ) <EOL> elif site_type == HC . SITE_TYPE_PIXIV_ARTIST_ID : <EOL> return GalleryPixivArtistID ( ) <EOL> elif site_type == HC . SITE_TYPE_PIXIV_TAG : <EOL> return GalleryPixivTag ( ) <EOL> elif site_type == HC . SITE_TYPE_TUMBLR : <EOL> return GalleryTumblr ( ) <EOL> _8CHAN_BOARDS_TO_MEDIA_HOSTS = { } <EOL> def GetImageboardThreadURLs ( thread_url ) : <EOL> ( thread_url , host , board , thread_id ) = ParseImageboardThreadURL ( thread_url ) <EOL> is_4chan = '<STR_LIT>' in host <EOL> is_8chan = '<STR_LIT>' in host <EOL> if is_4chan : <EOL> json_url = '<STR_LIT>' + board + '<STR_LIT>' + thread_id + '<STR_LIT>' <EOL> file_base = '<STR_LIT>' + board + '<STR_LIT:/>' <EOL> elif is_8chan : <EOL> json_url = '<STR_LIT>' + board + '<STR_LIT>' + thread_id + '<STR_LIT>' <EOL> html_url = '<STR_LIT>' + board + '<STR_LIT>' + thread_id + '<STR_LIT>' <EOL> if board not in _8CHAN_BOARDS_TO_MEDIA_HOSTS : <EOL> try : <EOL> response = ClientNetworking . RequestsGet ( html_url ) <EOL> thread_html = response . content <EOL> soup = GetSoup ( thread_html ) <EOL> file_infos = soup . find_all ( '<STR_LIT:p>' , class_ = "<STR_LIT>" ) <EOL> example_file_url = file_infos [ <NUM_LIT:0> ] . find ( '<STR_LIT:a>' ) [ '<STR_LIT>' ] <EOL> parse_result = urlparse . urlparse ( example_file_url ) <EOL> hostname = parse_result . hostname <EOL> if hostname is None : <EOL> hostname = '<STR_LIT>' <EOL> _8CHAN_BOARDS_TO_MEDIA_HOSTS [ board ] = hostname <EOL> except Exception as e : <EOL> _8CHAN_BOARDS_TO_MEDIA_HOSTS [ board ] = '<STR_LIT>' <EOL> media_host = _8CHAN_BOARDS_TO_MEDIA_HOSTS [ board ] <EOL> file_base = '<STR_LIT>' + media_host + '<STR_LIT:/>' + board + '<STR_LIT>' <EOL> return ( json_url , file_base ) <EOL> def GetSoup ( html ) : <EOL> return bs4 . BeautifulSoup ( html , '<STR_LIT>' ) <EOL> def GetYoutubeFormats ( youtube_url ) : <EOL> try : p = pafy . new ( youtube_url ) <EOL> except Exception as e : <EOL> raise Exception ( '<STR_LIT>' + os . linesep + HydrusData . ToUnicode ( e ) ) <EOL> info = { ( s . extension , s . resolution ) : ( s . url , s . title ) for s in p . streams if s . extension in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) } <EOL> return info <EOL> def THREADDownloadURL ( job_key , url , url_string ) : <EOL> job_key . SetVariable ( '<STR_LIT>' , url_string + '<STR_LIT>' ) <EOL> ( os_file_handle , temp_path ) = HydrusPaths . GetTempPath ( ) <EOL> try : <EOL> response = ClientNetworking . RequestsGet ( url , stream = True ) <EOL> if '<STR_LIT>' in response . headers : <EOL> gauge_range = int ( response . headers [ '<STR_LIT>' ] ) <EOL> else : <EOL> gauge_range = None <EOL> gauge_value = <NUM_LIT:0> <EOL> with open ( temp_path , '<STR_LIT:wb>' ) as f : <EOL> for chunk in response . iter_content ( chunk_size = <NUM_LIT> ) : <EOL> ( i_paused , should_quit ) = job_key . WaitIfNeeded ( ) <EOL> if should_quit : <EOL> return <EOL> f . write ( chunk ) <EOL> gauge_value += len ( chunk ) <EOL> if gauge_range is None : text = url_string + '<STR_LIT>' + HydrusData . ConvertIntToBytes ( gauge_value ) <EOL> else : text = url_string + '<STR_LIT>' + HydrusData . ConvertValueRangeToBytes ( gauge_value , gauge_range ) <EOL> job_key . SetVariable ( '<STR_LIT>' , text ) <EOL> job_key . SetVariable ( '<STR_LIT>' , ( gauge_value , gauge_range ) ) <EOL> job_key . DeleteVariable ( '<STR_LIT>' ) <EOL> job_key . SetVariable ( '<STR_LIT>' , '<STR_LIT>' + url_string ) <EOL> ( result , hash ) = HydrusGlobals . client_controller . WriteSynchronous ( '<STR_LIT>' , temp_path ) <EOL> except HydrusExceptions . NetworkException : <EOL> job_key . Cancel ( ) <EOL> raise <EOL> finally : <EOL> HydrusPaths . CleanUpTempPath ( os_file_handle , temp_path ) <EOL> if result in ( CC . STATUS_SUCCESSFUL , CC . STATUS_REDUNDANT ) : <EOL> if result == CC . STATUS_SUCCESSFUL : <EOL> job_key . SetVariable ( '<STR_LIT>' , url_string ) <EOL> else : <EOL> job_key . SetVariable ( '<STR_LIT>' , url_string + '<STR_LIT>' ) <EOL> job_key . SetVariable ( '<STR_LIT>' , { hash } ) <EOL> elif result == CC . STATUS_DELETED : <EOL> job_key . SetVariable ( '<STR_LIT>' , url_string + '<STR_LIT>' ) <EOL> job_key . Finish ( ) <EOL> def THREADDownloadURLs ( job_key , urls , title ) : <EOL> job_key . SetVariable ( '<STR_LIT>' , title + '<STR_LIT>' ) <EOL> num_successful = <NUM_LIT:0> <EOL> num_redundant = <NUM_LIT:0> <EOL> num_deleted = <NUM_LIT:0> <EOL> num_failed = <NUM_LIT:0> <EOL> successful_hashes = set ( ) <EOL> for ( i , url ) in enumerate ( urls ) : <EOL> ( i_paused , should_quit ) = job_key . WaitIfNeeded ( ) <EOL> if should_quit : <EOL> break <EOL> job_key . SetVariable ( '<STR_LIT>' , title + '<STR_LIT>' + HydrusData . ConvertValueRangeToPrettyString ( i + <NUM_LIT:1> , len ( urls ) ) ) <EOL> job_key . SetVariable ( '<STR_LIT>' , ( i + <NUM_LIT:1> , len ( urls ) ) ) <EOL> ( os_file_handle , temp_path ) = HydrusPaths . GetTempPath ( ) <EOL> try : <EOL> response = ClientNetworking . RequestsGet ( url , stream = True ) <EOL> if '<STR_LIT>' in response . headers : <EOL> gauge_range = int ( response . headers [ '<STR_LIT>' ] ) <EOL> else : <EOL> gauge_range = None <EOL> gauge_value = <NUM_LIT:0> <EOL> with open ( temp_path , '<STR_LIT:wb>' ) as f : <EOL> for chunk in response . iter_content ( chunk_size = <NUM_LIT> ) : <EOL> ( i_paused , should_quit ) = job_key . WaitIfNeeded ( ) <EOL> if should_quit : <EOL> return <EOL> f . write ( chunk ) <EOL> gauge_value += len ( chunk ) <EOL> if gauge_range is None : text = '<STR_LIT>' + HydrusData . ConvertIntToBytes ( gauge_value ) <EOL> else : text = '<STR_LIT>' + HydrusData . ConvertValueRangeToBytes ( gauge_value , gauge_range ) <EOL> job_key . SetVariable ( '<STR_LIT>' , text ) <EOL> job_key . SetVariable ( '<STR_LIT>' , ( gauge_value , gauge_range ) ) <EOL> job_key . SetVariable ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ( result , hash ) = HydrusGlobals . client_controller . WriteSynchronous ( '<STR_LIT>' , temp_path ) <EOL> except HydrusExceptions . NetworkException : <EOL> job_key . Cancel ( ) <EOL> raise <EOL> except Exception as e : <EOL> HydrusData . Print ( url + '<STR_LIT>' ) <EOL> HydrusData . PrintException ( e ) <EOL> num_failed += <NUM_LIT:1> <EOL> continue <EOL> finally : <EOL> HydrusPaths . CleanUpTempPath ( os_file_handle , temp_path ) <EOL> if result in ( CC . STATUS_SUCCESSFUL , CC . STATUS_REDUNDANT ) : <EOL> if result == CC . STATUS_SUCCESSFUL : <EOL> num_successful += <NUM_LIT:1> <EOL> else : <EOL> num_redundant += <NUM_LIT:1> <EOL> successful_hashes . add ( hash ) <EOL> elif result == CC . STATUS_DELETED : <EOL> num_deleted += <NUM_LIT:1> <EOL> text_components = [ ] <EOL> if num_successful > <NUM_LIT:0> : <EOL> text_components . append ( HydrusData . ConvertIntToPrettyString ( num_successful ) + '<STR_LIT>' ) <EOL> if num_redundant > <NUM_LIT:0> : <EOL> text_components . append ( HydrusData . ConvertIntToPrettyString ( num_redundant ) + '<STR_LIT>' ) <EOL> if num_deleted > <NUM_LIT:0> : <EOL> text_components . append ( HydrusData . ConvertIntToPrettyString ( num_deleted ) + '<STR_LIT>' ) <EOL> if num_failed > <NUM_LIT:0> : <EOL> text_components . append ( HydrusData . ConvertIntToPrettyString ( num_failed ) + '<STR_LIT>' ) <EOL> job_key . SetVariable ( '<STR_LIT>' , title + '<STR_LIT>' + '<STR_LIT:U+002CU+0020>' . join ( text_components ) ) <EOL> if len ( successful_hashes ) > <NUM_LIT:0> : <EOL> job_key . SetVariable ( '<STR_LIT>' , successful_hashes ) <EOL> job_key . DeleteVariable ( '<STR_LIT>' ) <EOL> job_key . DeleteVariable ( '<STR_LIT>' ) <EOL> job_key . DeleteVariable ( '<STR_LIT>' ) <EOL> job_key . Finish ( ) <EOL> def Parse4chanPostScreen ( html ) : <EOL> soup = GetSoup ( html ) <EOL> title_tag = soup . find ( '<STR_LIT:title>' ) <EOL> if title_tag . string == '<STR_LIT>' : return ( '<STR_LIT:success>' , None ) <EOL> elif title_tag . string == '<STR_LIT>' : <EOL> HydrusData . Print ( soup ) <EOL> text = '<STR_LIT>' <EOL> HydrusData . ShowText ( text ) <EOL> return ( '<STR_LIT>' , text ) <EOL> else : <EOL> try : <EOL> problem_tag = soup . find ( id = '<STR_LIT>' ) <EOL> if problem_tag is None : <EOL> HydrusData . Print ( soup ) <EOL> text = '<STR_LIT>' <EOL> HydrusData . ShowText ( text ) <EOL> return ( '<STR_LIT:error>' , text ) <EOL> problem = HydrusData . ToUnicode ( problem_tag ) <EOL> if '<STR_LIT>' in problem : return ( '<STR_LIT>' , None ) <EOL> elif '<STR_LIT>' in problem : return ( '<STR_LIT>' , None ) <EOL> elif '<STR_LIT>' in problem : return ( '<STR_LIT:error>' , '<STR_LIT>' ) <EOL> else : return ( '<STR_LIT:error>' , problem ) <EOL> except : return ( '<STR_LIT:error>' , '<STR_LIT>' ) <EOL> def ParseImageboardThreadURL ( thread_url ) : <EOL> try : <EOL> if '<STR_LIT:#>' in thread_url : <EOL> ( thread_url , post_anchor_gumpf ) = thread_url . split ( '<STR_LIT:#>' , <NUM_LIT:1> ) <EOL> parse_result = urlparse . urlparse ( thread_url ) <EOL> host = parse_result . hostname <EOL> request = parse_result . path <EOL> if host is None or request is None : <EOL> raise Exception ( ) <EOL> except : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> is_4chan = '<STR_LIT>' in host <EOL> is_8chan = '<STR_LIT>' in host <EOL> if not ( is_4chan or is_8chan ) : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> try : <EOL> if is_4chan : <EOL> ( board , rest_of_request ) = request [ <NUM_LIT:1> : ] . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> if '<STR_LIT:/>' in rest_of_request : <EOL> ( thread_id , gumpf ) = rest_of_request . split ( '<STR_LIT:/>' ) <EOL> else : <EOL> thread_id = rest_of_request <EOL> elif is_8chan : <EOL> ( board , rest_of_request ) = request [ <NUM_LIT:1> : ] . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> thread_id = rest_of_request [ : - <NUM_LIT:5> ] <EOL> except Exception as e : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> return ( thread_url , host , board , thread_id ) <EOL> def ParsePageForURLs ( html , starting_url ) : <EOL> soup = GetSoup ( html ) <EOL> all_links = soup . find_all ( '<STR_LIT:a>' ) <EOL> links_with_images = [ link for link in all_links if len ( link . find_all ( '<STR_LIT>' ) ) > <NUM_LIT:0> ] <EOL> urls = [ urlparse . urljoin ( starting_url , link [ '<STR_LIT>' ] ) for link in links_with_images ] <EOL> return urls <EOL> class GalleryIdentifier ( HydrusSerialisable . SerialisableBase ) : <EOL> SERIALISABLE_TYPE = HydrusSerialisable . SERIALISABLE_TYPE_GALLERY_IDENTIFIER <EOL> SERIALISABLE_VERSION = <NUM_LIT:1> <EOL> def __init__ ( self , site_type = None , additional_info = None ) : <EOL> HydrusSerialisable . SerialisableBase . __init__ ( self ) <EOL> self . _site_type = site_type <EOL> self . _additional_info = additional_info <EOL> def __eq__ ( self , other ) : <EOL> return self . __hash__ ( ) == other . __hash__ ( ) <EOL> def __hash__ ( self ) : <EOL> return ( self . _site_type , self . _additional_info ) . __hash__ ( ) <EOL> def __ne__ ( self , other ) : <EOL> return self . __hash__ ( ) != other . __hash__ ( ) <EOL> def __repr__ ( self ) : <EOL> text = '<STR_LIT>' + HC . site_type_string_lookup [ self . _site_type ] <EOL> if self . _site_type == HC . SITE_TYPE_BOORU : <EOL> text += '<STR_LIT>' + HydrusData . ToUnicode ( self . _additional_info ) <EOL> return text <EOL> def _GetSerialisableInfo ( self ) : <EOL> return ( self . _site_type , self . _additional_info ) <EOL> def _InitialiseFromSerialisableInfo ( self , serialisable_info ) : <EOL> ( self . _site_type , self . _additional_info ) = serialisable_info <EOL> def GetAdditionalInfo ( self ) : <EOL> return self . _additional_info <EOL> def GetSiteType ( self ) : <EOL> return self . _site_type <EOL> def ToString ( self ) : <EOL> text = HC . site_type_string_lookup [ self . _site_type ] <EOL> if self . _site_type == HC . SITE_TYPE_BOORU and self . _additional_info is not None : <EOL> booru_name = self . _additional_info <EOL> text = HC . site_type_string_lookup [ self . _site_type ] + '<STR_LIT>' + booru_name <EOL> return text <EOL> HydrusSerialisable . SERIALISABLE_TYPES_TO_OBJECT_TYPES [ HydrusSerialisable . SERIALISABLE_TYPE_GALLERY_IDENTIFIER ] = GalleryIdentifier <EOL> class Gallery ( object ) : <EOL> def _AddSessionCookies ( self , request_headers ) : pass <EOL> def _FetchData ( self , url , referer_url = None , report_hooks = None , temp_path = None ) : <EOL> if report_hooks is None : report_hooks = [ ] <EOL> request_headers = { } <EOL> if referer_url is not None : <EOL> request_headers [ '<STR_LIT>' ] = referer_url <EOL> self . _AddSessionCookies ( request_headers ) <EOL> return HydrusGlobals . client_controller . DoHTTP ( HC . GET , url , request_headers = request_headers , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> raise NotImplementedError ( ) <EOL> def _ParseGalleryPage ( self , data , url ) : <EOL> raise NotImplementedError ( ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> self . _FetchData ( url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetFileAndTags ( self , temp_path , url , report_hooks = None ) : <EOL> temp_path = self . GetFile ( temp_path , url , report_hooks ) <EOL> tags = self . GetTags ( url , report_hooks = report_hooks ) <EOL> return tags <EOL> def GetPage ( self , query , page_index ) : <EOL> gallery_url = self . _GetGalleryPageURL ( query , page_index ) <EOL> data = self . _FetchData ( gallery_url ) <EOL> ( page_of_urls , definitely_no_more_pages ) = self . _ParseGalleryPage ( data , gallery_url ) <EOL> return ( page_of_urls , definitely_no_more_pages ) <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> raise NotImplementedError ( ) <EOL> class GalleryBooru ( Gallery ) : <EOL> def __init__ ( self , booru_name ) : <EOL> try : <EOL> self . _booru = HydrusGlobals . client_controller . Read ( '<STR_LIT>' , booru_name ) <EOL> except : <EOL> raise Exception ( '<STR_LIT>' + booru_name + '<STR_LIT>' ) <EOL> self . _gallery_advance_num = None <EOL> ( self . _search_url , self . _advance_by_page_num , self . _search_separator , self . _thumb_classname ) = self . _booru . GetGalleryParsingInfo ( ) <EOL> Gallery . __init__ ( self ) <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> if self . _advance_by_page_num : <EOL> url_index = page_index + <NUM_LIT:1> <EOL> else : <EOL> if self . _gallery_advance_num is None : <EOL> if page_index == <NUM_LIT:0> : <EOL> url_index = page_index <EOL> else : <EOL> self . GetPage ( query , <NUM_LIT:0> ) <EOL> if self . _gallery_advance_num is None : <EOL> raise Exception ( '<STR_LIT>' ) <EOL> else : <EOL> url_index = page_index * self . _gallery_advance_num <EOL> else : <EOL> url_index = page_index * self . _gallery_advance_num <EOL> tags = query . split ( '<STR_LIT:U+0020>' ) <EOL> if '<STR_LIT>' in self . _search_url : <EOL> tags_to_use = [ ] <EOL> for tag in tags : <EOL> if '<STR_LIT:/>' in tag : <EOL> tag = tag . replace ( '<STR_LIT:/>' , '<STR_LIT>' ) <EOL> tags_to_use . append ( tag ) <EOL> tags = tags_to_use <EOL> tags_replace = self . _search_separator . join ( [ urllib . quote ( HydrusData . ToByteString ( tag ) , '<STR_LIT>' ) for tag in tags ] ) <EOL> return self . _search_url . replace ( '<STR_LIT>' , tags_replace ) . replace ( '<STR_LIT>' , str ( url_index ) ) <EOL> def _ParseGalleryPage ( self , html , url_base ) : <EOL> definitely_no_more_pages = False <EOL> urls_set = set ( ) <EOL> urls = [ ] <EOL> soup = GetSoup ( html ) <EOL> def starts_with_classname ( classname ) : return classname is not None and classname . startswith ( self . _thumb_classname ) <EOL> thumbnails = soup . find_all ( class_ = starts_with_classname ) <EOL> popular_thumbnail_parent = soup . find ( id = '<STR_LIT>' ) <EOL> if popular_thumbnail_parent is not None : <EOL> popular_thumbnails = popular_thumbnail_parent . find_all ( class_ = starts_with_classname ) <EOL> thumbnails = thumbnails [ len ( popular_thumbnails ) : ] <EOL> if self . _gallery_advance_num is None : <EOL> if len ( thumbnails ) == <NUM_LIT:0> : <EOL> definitely_no_more_pages = True <EOL> else : <EOL> self . _gallery_advance_num = len ( thumbnails ) <EOL> for thumbnail in thumbnails : <EOL> links = thumbnail . find_all ( '<STR_LIT:a>' ) <EOL> if thumbnail . name == '<STR_LIT:a>' : links . append ( thumbnail ) <EOL> for link in links : <EOL> if link . string is not None and link . string == '<STR_LIT>' : continue <EOL> url = link [ '<STR_LIT>' ] <EOL> url = urlparse . urljoin ( url_base , url ) <EOL> if url not in urls_set : <EOL> urls_set . add ( url ) <EOL> urls . append ( url ) <EOL> return ( urls , definitely_no_more_pages ) <EOL> def _ParseImagePage ( self , html , url_base ) : <EOL> ( search_url , search_separator , advance_by_page_num , thumb_classname , image_id , image_data , tag_classnames_to_namespaces ) = self . _booru . GetData ( ) <EOL> soup = GetSoup ( html ) <EOL> image_base = None <EOL> image_url = None <EOL> try : <EOL> if image_id is not None : <EOL> image = soup . find ( id = image_id ) <EOL> if image is None : <EOL> image_string = soup . find ( text = re . compile ( '<STR_LIT>' ) ) <EOL> if image_string is None : image_string = soup . find ( text = re . compile ( '<STR_LIT>' ) ) <EOL> if image_string is None : <EOL> if image_url is None : <EOL> a_tags = soup . find_all ( '<STR_LIT:a>' ) <EOL> for a_tag in a_tags : <EOL> href = a_tag [ '<STR_LIT>' ] <EOL> if href is not None : <EOL> if href . endswith ( '<STR_LIT>' ) : <EOL> image_url = href <EOL> break <EOL> else : <EOL> image = image_string . parent <EOL> image_url = image [ '<STR_LIT>' ] <EOL> else : <EOL> if image . name in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> image_url = image [ '<STR_LIT:src>' ] <EOL> if '<STR_LIT>' in image_url : <EOL> image = soup . find ( id = '<STR_LIT>' ) <EOL> image_url = image [ '<STR_LIT>' ] <EOL> elif image . name == '<STR_LIT:a>' : <EOL> image_url = image [ '<STR_LIT>' ] <EOL> if image_data is not None : <EOL> links = soup . find_all ( '<STR_LIT:a>' ) <EOL> ok_link = None <EOL> better_link = None <EOL> for link in links : <EOL> if link . string is not None : <EOL> if link . string . startswith ( image_data ) : <EOL> ok_link = link [ '<STR_LIT>' ] <EOL> if link . string . startswith ( '<STR_LIT>' ) : <EOL> better_link = link [ '<STR_LIT>' ] <EOL> break <EOL> if better_link is not None : <EOL> image_url = better_link <EOL> else : <EOL> image_url = ok_link <EOL> except Exception as e : <EOL> raise HydrusExceptions . NetworkException ( '<STR_LIT>' + url_base + '<STR_LIT:!>' + os . linesep + HydrusData . ToUnicode ( e ) ) <EOL> if image_url is None : <EOL> raise HydrusExceptions . NetworkException ( '<STR_LIT>' + url_base + '<STR_LIT:!>' ) <EOL> image_url = urlparse . urljoin ( url_base , image_url ) <EOL> tags = [ ] <EOL> for ( tag_classname , namespace ) in tag_classnames_to_namespaces . items ( ) : <EOL> tag_list_entries = soup . find_all ( class_ = tag_classname ) <EOL> for tag_list_entry in tag_list_entries : <EOL> links = tag_list_entry . find_all ( '<STR_LIT:a>' ) <EOL> if tag_list_entry . name == '<STR_LIT:a>' : links . append ( tag_list_entry ) <EOL> for link in links : <EOL> if link . string not in ( '<STR_LIT:?>' , '<STR_LIT:->' , '<STR_LIT:+>' ) : <EOL> if namespace == '<STR_LIT>' : tags . append ( link . string ) <EOL> else : tags . append ( namespace + '<STR_LIT::>' + link . string ) <EOL> return ( image_url , tags ) <EOL> def _GetFileURLAndTags ( self , url , report_hooks = None ) : <EOL> html = self . _FetchData ( url , report_hooks = report_hooks ) <EOL> return self . _ParseImagePage ( html , url ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetFileAndTags ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> return tags <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> return tags <EOL> class GalleryDeviantArt ( Gallery ) : <EOL> def _AddSessionCookies ( self , request_headers ) : <EOL> manager = HydrusGlobals . client_controller . GetManager ( '<STR_LIT>' ) <EOL> cookies = manager . GetCookies ( '<STR_LIT>' ) <EOL> ClientNetworking . AddCookiesToHeaders ( cookies , request_headers ) <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist = query <EOL> return '<STR_LIT>' + artist + '<STR_LIT>' + str ( page_index * <NUM_LIT> ) <EOL> def _ParseGalleryPage ( self , html , url_base ) : <EOL> definitely_no_more_pages = False <EOL> urls = [ ] <EOL> soup = GetSoup ( html ) <EOL> thumbs_container = soup . find ( class_ = '<STR_LIT>' ) <EOL> artist = url_base . split ( '<STR_LIT>' ) [ <NUM_LIT:1> ] . split ( '<STR_LIT>' ) [ <NUM_LIT:0> ] <EOL> links = thumbs_container . find_all ( '<STR_LIT:a>' , class_ = '<STR_LIT>' ) <EOL> for link in links : <EOL> url = link [ '<STR_LIT>' ] <EOL> urls . append ( url ) <EOL> tags = [ ] <EOL> tags . append ( '<STR_LIT>' + artist ) <EOL> try : <EOL> raw_title = link [ '<STR_LIT:title>' ] <EOL> raw_title_reversed = raw_title [ : : - <NUM_LIT:1> ] <EOL> ( creator_and_gumpf_reversed , title_reversed ) = raw_title_reversed . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> title = title_reversed [ : : - <NUM_LIT:1> ] <EOL> tags . append ( '<STR_LIT>' + title ) <EOL> except : pass <EOL> SetExtraURLInfo ( url , tags ) <EOL> return ( urls , definitely_no_more_pages ) <EOL> def _ParseImagePage ( self , html , referer_url ) : <EOL> soup = GetSoup ( html ) <EOL> download_button = soup . find ( '<STR_LIT:a>' , class_ = '<STR_LIT>' ) <EOL> if download_button is None : <EOL> img = soup . find ( class_ = '<STR_LIT>' ) <EOL> if img is None : <EOL> a_tumblr = soup . find ( id = '<STR_LIT>' ) <EOL> tumblr_url = a_tumblr [ '<STR_LIT>' ] <EOL> parse_result = urlparse . urlparse ( tumblr_url ) <EOL> query_parse_result = urlparse . parse_qs ( parse_result . query ) <EOL> img_url = query_parse_result [ '<STR_LIT:source>' ] [ <NUM_LIT:0> ] <EOL> else : <EOL> img_url = img [ '<STR_LIT:src>' ] <EOL> else : <EOL> img_url = download_button [ '<STR_LIT>' ] <EOL> return img_url <EOL> def _GetFileURL ( self , url , report_hooks = None ) : <EOL> html = self . _FetchData ( url , report_hooks = report_hooks ) <EOL> return self . _ParseImagePage ( html , url ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> file_url = self . _GetFileURL ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> result = GetExtraURLInfo ( url ) <EOL> if result is None : <EOL> return [ ] <EOL> else : <EOL> return result <EOL> class GalleryGiphy ( Gallery ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> tag = query <EOL> return '<STR_LIT>' + urllib . quote ( HydrusData . ToByteString ( tag ) . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:+>' ) , '<STR_LIT>' ) + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> def _ParseGalleryPage ( self , data , url_base ) : <EOL> definitely_no_more_pages = False <EOL> json_dict = json . loads ( data ) <EOL> urls = [ ] <EOL> if '<STR_LIT:data>' in json_dict : <EOL> json_data = json_dict [ '<STR_LIT:data>' ] <EOL> for d in json_data : <EOL> url = d [ '<STR_LIT>' ] <EOL> id = d [ '<STR_LIT:id>' ] <EOL> SetExtraURLInfo ( url , id ) <EOL> urls . append ( url ) <EOL> return ( urls , definitely_no_more_pages ) <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> id = GetExtraURLInfo ( url ) <EOL> if id is None : <EOL> return [ ] <EOL> else : <EOL> url = '<STR_LIT>' + str ( id ) <EOL> try : <EOL> raw_json = self . _FetchData ( url , report_hooks = report_hooks ) <EOL> json_dict = json . loads ( raw_json ) <EOL> tags_data = json_dict [ '<STR_LIT:data>' ] [ '<STR_LIT>' ] <EOL> return [ tag_data [ '<STR_LIT:name>' ] for tag_data in tags_data ] <EOL> except Exception as e : <EOL> HydrusData . ShowException ( e ) <EOL> return [ ] <EOL> class GalleryHentaiFoundry ( Gallery ) : <EOL> def _AddSessionCookies ( self , request_headers ) : <EOL> manager = HydrusGlobals . client_controller . GetManager ( '<STR_LIT>' ) <EOL> cookies = manager . GetCookies ( '<STR_LIT>' ) <EOL> ClientNetworking . AddCookiesToHeaders ( cookies , request_headers ) <EOL> def _GetFileURLAndTags ( self , url , report_hooks = None ) : <EOL> html = self . _FetchData ( url , report_hooks = report_hooks ) <EOL> return self . _ParseImagePage ( html , url ) <EOL> def _ParseGalleryPage ( self , html , url_base ) : <EOL> definitely_no_more_pages = False <EOL> urls_set = set ( ) <EOL> soup = GetSoup ( html ) <EOL> def correct_url ( href ) : <EOL> if href is None : <EOL> return False <EOL> if href . count ( '<STR_LIT:/>' ) == <NUM_LIT:5> and href . startswith ( '<STR_LIT>' ) : <EOL> ( nothing , pictures , user , artist_name , file_id , title ) = href . split ( '<STR_LIT:/>' ) <EOL> if file_id != '<STR_LIT>' : return True <EOL> return False <EOL> urls = [ ] <EOL> links = soup . find_all ( '<STR_LIT:a>' , href = correct_url ) <EOL> for link in links : <EOL> url = '<STR_LIT>' + link [ '<STR_LIT>' ] <EOL> if url not in urls_set : <EOL> urls_set . add ( url ) <EOL> urls . append ( url ) <EOL> if '<STR_LIT>' not in html : <EOL> definitely_no_more_pages = True <EOL> return ( urls , definitely_no_more_pages ) <EOL> def _ParseImagePage ( self , html , url_base ) : <EOL> try : <EOL> index = html . index ( '<STR_LIT>' ) <EOL> image_url = html [ index : index + <NUM_LIT> ] <EOL> if '<STR_LIT:">' in image_url : ( image_url , gumpf ) = image_url . split ( '<STR_LIT:">' , <NUM_LIT:1> ) <EOL> if '<STR_LIT>' in image_url : ( image_url , gumpf ) = image_url . split ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> image_url = '<STR_LIT>' + image_url <EOL> except Exception as e : <EOL> raise Exception ( '<STR_LIT>' + os . linesep + HydrusData . ToUnicode ( e ) ) <EOL> soup = GetSoup ( html ) <EOL> tags = [ ] <EOL> try : <EOL> title = soup . find ( '<STR_LIT:title>' ) <EOL> ( data , nothing ) = title . string . split ( '<STR_LIT>' ) <EOL> data_reversed = data [ : : - <NUM_LIT:1> ] <EOL> ( artist_reversed , title_reversed ) = data_reversed . split ( '<STR_LIT>' ) <EOL> artist = artist_reversed [ : : - <NUM_LIT:1> ] <EOL> title = title_reversed [ : : - <NUM_LIT:1> ] <EOL> tags . append ( '<STR_LIT>' + artist ) <EOL> tags . append ( '<STR_LIT>' + title ) <EOL> except : pass <EOL> tag_links = soup . find_all ( '<STR_LIT:a>' , rel = '<STR_LIT>' ) <EOL> for tag_link in tag_links : tags . append ( tag_link . string ) <EOL> return ( image_url , tags ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetFileAndTags ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> return tags <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> return tags <EOL> class GalleryHentaiFoundryArtistPictures ( GalleryHentaiFoundry ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist = query <EOL> gallery_url = '<STR_LIT>' + artist <EOL> return gallery_url + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> class GalleryHentaiFoundryArtistScraps ( GalleryHentaiFoundry ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist = query <EOL> gallery_url = '<STR_LIT>' + artist + '<STR_LIT>' <EOL> return gallery_url + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> class GalleryHentaiFoundryTags ( GalleryHentaiFoundry ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> tags = query . split ( '<STR_LIT:U+0020>' ) <EOL> return '<STR_LIT>' + '<STR_LIT:+>' . join ( tags ) + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> class GalleryNewgrounds ( Gallery ) : <EOL> def _GetFileURLAndTags ( self , url , report_hooks = None ) : <EOL> html = self . _FetchData ( url , report_hooks = report_hooks ) <EOL> return self . _ParseImagePage ( html , url ) <EOL> def _ParseGalleryPage ( self , html , url_base ) : <EOL> soup = GetSoup ( html ) <EOL> fatcol = soup . find ( '<STR_LIT>' , class_ = '<STR_LIT>' ) <EOL> links = fatcol . find_all ( '<STR_LIT:a>' ) <EOL> urls_set = set ( ) <EOL> urls = [ ] <EOL> for link in links : <EOL> try : <EOL> url = link [ '<STR_LIT>' ] <EOL> if url not in urls_set : <EOL> if url . startswith ( '<STR_LIT>' ) : <EOL> urls_set . add ( url ) <EOL> urls . append ( url ) <EOL> except : pass <EOL> definitely_no_more_pages = True <EOL> return ( urls , definitely_no_more_pages ) <EOL> def _ParseImagePage ( self , html , url_base ) : <EOL> soup = GetSoup ( html ) <EOL> tags = set ( ) <EOL> author_links = soup . find ( '<STR_LIT>' , class_ = '<STR_LIT>' ) <EOL> if author_links is not None : <EOL> authors = set ( ) <EOL> links = author_links . find_all ( '<STR_LIT:a>' ) <EOL> for link in links : <EOL> try : <EOL> href = link [ '<STR_LIT>' ] <EOL> creator = href . replace ( '<STR_LIT>' , '<STR_LIT>' ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> tags . add ( u'<STR_LIT>' + creator ) <EOL> except : pass <EOL> try : <EOL> title = soup . find ( '<STR_LIT:title>' ) <EOL> tags . add ( u'<STR_LIT>' + title . string ) <EOL> except : pass <EOL> all_links = soup . find_all ( '<STR_LIT:a>' ) <EOL> for link in all_links : <EOL> try : <EOL> href = link [ '<STR_LIT>' ] <EOL> if '<STR_LIT>' in href : tags . add ( link . string ) <EOL> except : pass <EOL> flash_url = html . split ( '<STR_LIT>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> flash_url = flash_url . split ( '<STR_LIT:">' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] <EOL> flash_url = flash_url . replace ( "<STR_LIT>" , '<STR_LIT:/>' ) <EOL> flash_url = '<STR_LIT>' + flash_url <EOL> return ( flash_url , tags ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetFileAndTags ( self , temp_path , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( file_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> return tags <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> ( file_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> return tags <EOL> class GalleryNewgroundsGames ( GalleryNewgrounds ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist = query <EOL> return '<STR_LIT>' + artist + '<STR_LIT>' <EOL> class GalleryNewgroundsMovies ( GalleryNewgrounds ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist = query <EOL> return '<STR_LIT>' + artist + '<STR_LIT>' <EOL> class GalleryPixiv ( Gallery ) : <EOL> def _AddSessionCookies ( self , request_headers ) : <EOL> manager = HydrusGlobals . client_controller . GetManager ( '<STR_LIT>' ) <EOL> cookies = manager . GetCookies ( '<STR_LIT>' ) <EOL> ClientNetworking . AddCookiesToHeaders ( cookies , request_headers ) <EOL> def _ParseGalleryPage ( self , html , url_base ) : <EOL> definitely_no_more_pages = False <EOL> urls = [ ] <EOL> soup = GetSoup ( html ) <EOL> manga_links = soup . find_all ( class_ = '<STR_LIT>' ) <EOL> thumbnail_links = soup . find_all ( class_ = '<STR_LIT>' ) <EOL> manga_urls = { manga_link [ '<STR_LIT>' ] for manga_link in manga_links } <EOL> thumbnail_urls = [ thumbnail_link [ '<STR_LIT>' ] for thumbnail_link in thumbnail_links ] <EOL> for thumbnail_url in thumbnail_urls : <EOL> if thumbnail_url in manga_urls : <EOL> pass <EOL> else : <EOL> url = urlparse . urljoin ( url_base , thumbnail_url ) <EOL> urls . append ( url ) <EOL> return ( urls , definitely_no_more_pages ) <EOL> def _ParseImagePage ( self , html , page_url ) : <EOL> if '<STR_LIT>' in html : <EOL> manga_url = page_url . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> raise HydrusExceptions . MimeException ( page_url + '<STR_LIT>' ) <EOL> if '<STR_LIT>' in html : <EOL> raise HydrusExceptions . MimeException ( page_url + '<STR_LIT>' ) <EOL> soup = GetSoup ( html ) <EOL> original_image = soup . find ( class_ = '<STR_LIT>' ) <EOL> image_url = original_image [ '<STR_LIT>' ] <EOL> tags_parent = soup . find ( '<STR_LIT>' , class_ = '<STR_LIT>' ) <EOL> tags = [ link . string for link in tags_parent . find_all ( '<STR_LIT:a>' , class_ = '<STR_LIT:text>' ) ] <EOL> user = soup . find ( '<STR_LIT>' , class_ = '<STR_LIT:user>' ) <EOL> tags . append ( '<STR_LIT>' + user . string ) <EOL> title_parent = soup . find ( '<STR_LIT>' , class_ = re . compile ( '<STR_LIT>' ) ) <EOL> title = title_parent . find ( '<STR_LIT>' , class_ = '<STR_LIT:title>' ) <EOL> tags . append ( '<STR_LIT>' + title . string ) <EOL> return ( image_url , tags ) <EOL> def _GetFileURLAndTags ( self , page_url , report_hooks = None ) : <EOL> html = self . _FetchData ( page_url , report_hooks = report_hooks ) <EOL> return self . _ParseImagePage ( html , page_url ) <EOL> def GetFile ( self , temp_path , url , report_hooks = None ) : <EOL> ( image_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( image_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> def GetFileAndTags ( self , temp_path , url , report_hooks = None ) : <EOL> ( image_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> self . _FetchData ( image_url , referer_url = url , report_hooks = report_hooks , temp_path = temp_path ) <EOL> return tags <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> ( image_url , tags ) = self . _GetFileURLAndTags ( url , report_hooks = report_hooks ) <EOL> return tags <EOL> class GalleryPixivArtistID ( GalleryPixiv ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> artist_id = query <EOL> gallery_url = '<STR_LIT>' + str ( artist_id ) <EOL> return gallery_url + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> class GalleryPixivTag ( GalleryPixiv ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> tag = query <EOL> gallery_url = '<STR_LIT>' + urllib . quote ( HydrusData . ToByteString ( tag ) , '<STR_LIT>' ) + '<STR_LIT>' <EOL> return gallery_url + '<STR_LIT>' + str ( page_index + <NUM_LIT:1> ) <EOL> class GalleryTumblr ( Gallery ) : <EOL> def _GetGalleryPageURL ( self , query , page_index ) : <EOL> username = query <EOL> return '<STR_LIT>' + username + '<STR_LIT>' + str ( page_index * <NUM_LIT:50> ) + '<STR_LIT>' <EOL> def _ParseGalleryPage ( self , data , url_base ) : <EOL> definitely_no_more_pages = False <EOL> processed_raw_json = data . split ( '<STR_LIT>' ) [ <NUM_LIT:1> ] [ : - <NUM_LIT:1> ] <EOL> json_object = json . loads ( processed_raw_json ) <EOL> urls = [ ] <EOL> if '<STR_LIT>' in json_object : <EOL> for post in json_object [ '<STR_LIT>' ] : <EOL> if '<STR_LIT>' in post : tags = post [ '<STR_LIT>' ] <EOL> else : tags = [ ] <EOL> post_type = post [ '<STR_LIT:type>' ] <EOL> if post_type == '<STR_LIT>' : <EOL> if len ( post [ '<STR_LIT>' ] ) == <NUM_LIT:0> : <EOL> try : <EOL> url = post [ '<STR_LIT>' ] <EOL> SetExtraURLInfo ( url , tags ) <EOL> urls . append ( url ) <EOL> except : pass <EOL> else : <EOL> for photo in post [ '<STR_LIT>' ] : <EOL> try : <EOL> url = photo [ '<STR_LIT>' ] <EOL> SetExtraURLInfo ( url , tags ) <EOL> urls . append ( url ) <EOL> except : pass <EOL> return ( urls , definitely_no_more_pages ) <EOL> def GetTags ( self , url , report_hooks = None ) : <EOL> result = GetExtraURLInfo ( url ) <EOL> if result is None : <EOL> return [ ] <EOL> else : <EOL> return result </s>
