title: using scrapy extracting data inside links
body: <p>I have been trying to extract data from consumercomplaints.in the title and the data inside those title links.I wrote the following code and unable to parse through the links and extract the data and also I am unable to extract all the links related.plz guide</p>
  
  <pre><code>from scrapy.contrib.spiders import CrawlSpider, Rule
  from scrapy.selector import Selector
  from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
  from urlparse import urljoin
  from comp.items import CompItem
  
  class criticspider(CrawlSpider):
      name ="comp"
      allowed_domains =["consumercomplaints.in"]
      #start_urls =["http://www.consumercomplaints.in/?search=delhivery&amp;page=2","http://www.consumercomplaints.in/?search=delhivery&amp;page=3","http://www.consumercomplaints.in/?search=delhivery&amp;page=4","http://www.consumercomplaints.in/?search=delhivery&amp;page=5","http://www.consumercomplaints.in/?search=delhivery&amp;page=6","http://www.consumercomplaints.in/?search=delhivery&amp;page=7","http://www.consumercomplaints.in/?search=delhivery&amp;page=8","http://www.consumercomplaints.in/?search=delhivery&amp;page=9","http://www.consumercomplaints.in/?search=delhivery&amp;page=10","http://www.consumercomplaints.in/?search=delhivery&amp;page=11"]
      start_urls=["http://www.consumercomplaints.in/?search=delhivery"]
      rules=(
          Rule(SgmlLinkExtractor(allow=("search=delhivery&amp;page=1/+",)), callback="parse", follow=True),
          #Rule(SgmlLinkExtractor(allow=("startrow=\d",)),callback="parse_health",follow=True),
      )
      def parse(self,response):
          hxs = Selector(response)
          sites = hxs.select('//table[@width="100%"]')
          items = []
  
          for site in sites:
              item = CompItem()
              item['title'] = site.select('.//td[@class="complaint"]/a/span/text()').extract()
              item['link'] = site.select('.//td[@class="complaint"]/a/@href').extract()
              if item['link']:
                  if 'http://' not in item['link']:
                      item['link'] = urljoin(response.url, item['link'])
                  yield Request(item['link'],
                      meta={'item': item},
                      callback=self.anchor_page)
  
              # item['intro'] = site.select('.//td[@class="small"]//a[2]/text()').extract()
              # item['heading'] = site.select('.//td[@class="compl-text"]/div/b[1]/text()').extract()
              # item['date'] = site.select('.//td[@class="small"]/text()[2]').extract()
              # item['complaint'] = site.select('.//td[@class="compl-text"]/div/text()').extract()
              items.append(item)
  
  
      def anchor_page(self, response):
          hxs = Selector(response)
          old_item = response.request.meta['item'] # Receiving parse Method item that was in Request meta
          # parse some more values
          #place them in old_item
          #e.g
          old_item['data']=hxs.select('.//td[@class="compl-text"]/div/text()').extract()
          yield old_item
  </code></pre>