lib/wmap/url_crawler/adware_tag.rb
# Give a  site, locate the landing page, then sift out the adware tag if found
Give a  site, locate the landing page, then sift out the adware tag if found
Give a site locate the landing page then sift out the adware tag if found
def check_adware(site,use_cache=true)
		  puts "Check the site for known Adware tags: #{site}" if @verbose
      record = Hash.new
			if use_cache && @tag_store.key?(site)
			  puts "Site entry already exist. Skipping: #{site}" if @verbose
			else
        url = fast_landing(site)
        if @landings.key?(url)
          record[site] = @landings[url]
          return record
        end
        tags = find_tags(url)
        return record if tags.size==0
        tag_vers=tags.map do |tag|
          get_ver(url,tag)
        end
        tag_descs=tags.map do |tag|
          Base64.urlsafe_encode64(get_desc(url,tag))
        end
				if tags
          record[site] = [url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
          @landings[url] = [url, tags.join("|"), tag_vers.join("|"), tag_descs.join("|")]
          @tag_store.merge!(record)
          puts "Tag entry loaded: #{record}" if @verbose
        else
          puts "No tag found. Skip site #{site}" if @verbose
        end
			end
      return record
    rescue => ee
			puts "Exception on method #{__method__}: #{ee}: #{site}" if @verbose
  	end
def check_adware ( site , use_cache = true ) puts "Check the site for known Adware tags: #{site}" if @verbose record = Hash . new if use_cache && @tag_store . key? ( site ) puts "Site entry already exist. Skipping: #{site}" if @verbose else url = fast_landing ( site ) if @landings . key? ( url ) record [ site ] = @landings [ url ] return record end tags = find_tags ( url ) return record if tags . size == 0 tag_vers = tags . map do | tag | get_ver ( url , tag ) end tag_descs = tags . map do | tag | Base64 . urlsafe_encode64 ( get_desc ( url , tag ) ) end if tags record [ site ] = [ url , tags . join ( "|" ) , tag_vers . join ( "|" ) , tag_descs . join ( "|" ) ] @landings [ url ] = [ url , tags . join ( "|" ) , tag_vers . join ( "|" ) , tag_descs . join ( "|" ) ] @tag_store . merge! ( record ) puts "Tag entry loaded: #{record}" if @verbose else puts "No tag found. Skip site #{site}" if @verbose end end return record rescue => ee puts "Exception on method #{__method__}: #{ee}: #{site}" if @verbose end
Give a  site, locate the landing page, then sift out the adware tag if found
Give a site locate the landing page then sift out the adware tag if found
site
<not_specific>
use_cache
<not_specific>