<s> from scrapy . contrib . spiders import CrawlSpider , Rule <EOL> from scrapy . contrib . linkextractors . sgml import SgmlLinkExtractor <EOL> from scrapy . selector import HtmlXPathSelector <EOL> from openrecipes . items import RecipeItem , RecipeItemLoader <EOL> from openrecipes . util import strip_html <EOL> class BBCgoodfoodMixin ( object ) : <EOL> source = '<STR_LIT>' <EOL> def remove_whitespace ( self , input ) : <EOL> return "<STR_LIT:U+0020>" . join ( "<STR_LIT>" . join ( input ) . split ( ) ) . strip ( ) <EOL> def parse_item ( self , response ) : <EOL> hxs = HtmlXPathSelector ( response ) <EOL> base_path = """<STR_LIT>""" <EOL> recipes_scopes = hxs . select ( base_path ) <EOL> name_path = '<STR_LIT>' <EOL> description_path = '<STR_LIT>' <EOL> image_path = '<STR_LIT>' <EOL> prepTime_path = '<STR_LIT>' <EOL> cookTime_path = '<STR_LIT>' <EOL> recipeYield_path = '<STR_LIT>' <EOL> recipeYieldAmount_path = '<STR_LIT>' <EOL> ingredients_path = '<STR_LIT>' <EOL> recipes = [ ] <EOL> for r_scope in recipes_scopes : <EOL> il = RecipeItemLoader ( item = RecipeItem ( ) ) <EOL> il . add_value ( '<STR_LIT:source>' , self . source ) <EOL> il . add_value ( '<STR_LIT:name>' , "<STR_LIT>" . join ( r_scope . select ( name_path ) . extract ( ) ) ) <EOL> il . add_value ( '<STR_LIT:url>' , response . url ) <EOL> base_img_url = '<STR_LIT:/>' . join ( response . url . split ( '<STR_LIT:/>' ) [ : - <NUM_LIT:1> ] ) <EOL> img_name = "<STR_LIT>" . join ( r_scope . select ( image_path ) . extract ( ) ) . strip ( ) <EOL> if img_name : <EOL> il . add_value ( '<STR_LIT:image>' , '<STR_LIT:/>' . join ( [ base_img_url , img_name ] ) ) <EOL> il . add_value ( '<STR_LIT:description>' , r_scope . select ( description_path ) . extract ( ) ) <EOL> prepSentence = "<STR_LIT:U+0020>" . join ( r_scope . select ( prepTime_path ) . extract ( ) ) . strip ( ) <EOL> if prepSentence : <EOL> prepSentence = self . remove_whitespace ( prepSentence ) <EOL> il . add_value ( '<STR_LIT>' , prepSentence . split ( '<STR_LIT:U+0020>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] ) <EOL> cookSentence = "<STR_LIT:U+0020>" . join ( r_scope . select ( cookTime_path ) . extract ( ) ) . strip ( ) <EOL> if cookSentence : <EOL> cookSentence = self . remove_whitespace ( cookSentence ) <EOL> il . add_value ( '<STR_LIT>' , cookSentence . split ( '<STR_LIT:U+0020>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] ) <EOL> if r_scope . select ( recipeYieldAmount_path ) . extract ( ) : <EOL> yieldAmount = r_scope . select ( recipeYieldAmount_path ) . extract ( ) [ <NUM_LIT:0> ] . strip ( ) <EOL> else : <EOL> yieldAmount = "<STR_LIT>" <EOL> yieldList = r_scope . select ( recipeYield_path ) . extract ( ) <EOL> if yieldList : <EOL> yieldString = yieldList [ <NUM_LIT:0> ] . strip ( ) <EOL> il . add_value ( '<STR_LIT>' , ( '<STR_LIT>' % ( yieldString , yieldAmount ) ) . strip ( ) ) <EOL> ingredient_scopes = r_scope . select ( ingredients_path ) <EOL> ingredients = [ ] <EOL> for i_scope in ingredient_scopes : <EOL> ingredient = strip_html ( i_scope . extract ( ) ) <EOL> ingredient = self . remove_whitespace ( ingredient ) <EOL> ingredients . append ( ingredient ) <EOL> il . add_value ( '<STR_LIT>' , ingredients ) <EOL> recipes . append ( il . load_item ( ) ) <EOL> return recipes <EOL> class BBCgoodfoodcrawlSpider ( CrawlSpider , BBCgoodfoodMixin ) : <EOL> name = "<STR_LIT>" <EOL> allowed_domains = [ "<STR_LIT>" ] <EOL> start_urls = [ <EOL> "<STR_LIT>" , <EOL> ] <EOL> rules = ( <EOL> Rule ( SgmlLinkExtractor ( allow = ( '<STR_LIT>' ) ) ) , <EOL> Rule ( SgmlLinkExtractor ( allow = ( '<STR_LIT>' ) ) ) , <EOL> Rule ( SgmlLinkExtractor ( allow = ( '<STR_LIT>' ) , <EOL> deny = '<STR_LIT>' ) , <EOL> callback = '<STR_LIT>' ) , <EOL> ) </s>
