lib/wmap/url_checker.rb
# Main worker method to perform various checks on the URL / site
Main worker method to perform various checks on the URL / site
Main worker method to perform various checks on the URL / site
def url_worker (url)
		puts "Checking out an unknown URL: #{url}" if @verbose
		url=url.strip.downcase
		raise "Invalid URL format: #{url}" unless is_url?(url)
		timestamp=Time.now
		host=url_2_host(url)
		ip=host_2_ip(host)
		port=url_2_port(url)
		code=10000
		if @url_code.key?(url)
			code=@url_code[url]
		else
			code=response_code(url)
		end
		if code>=300 && code < 400
			loc=landing_location(4,url)
		else
			loc=nil
		end
		if @url_finger_print.key?(url)
			fp=@url_finger_print[url]
		else
			fp=response_body_md5(url)
		end
		if @url_server.key?(url)
			server=@url_server[url]
		else
			server=get_server_header(url)
		end
		checker=Hash.new
		checker['ip']=ip
		checker['port']=port
		checker['url']=url
		checker['code']=code
		checker['redirection']=loc
		checker['md5']=fp
		checker['server']=server
		checker['timestamp']=timestamp
		if Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
			checker['status']="int_hosted"
		else
			checker['status']="ext_hosted"
		end
		return checker
	rescue OpenSSL::SSL::SSLError => es  
		checker=Hash.new
		checker['ip']=ip
		checker['port']=port
		checker['url']=url
		checker['code']=20000
		checker['server']="Unknown SSL error: #{es}"
		checker['md']=nil
		checker['redirection']=nil
		checker['timestamp']=timestamp
		return checker
	end
def url_worker ( url ) puts "Checking out an unknown URL: #{url}" if @verbose url = url . strip . downcase raise "Invalid URL format: #{url}" unless is_url? ( url ) timestamp = Time . now host = url_2_host ( url ) ip = host_2_ip ( host ) port = url_2_port ( url ) code = 10000 if @url_code . key? ( url ) code = @url_code [ url ] else code = response_code ( url ) end if code >= 300 && code < 400 loc = landing_location ( 4 , url ) else loc = nil end if @url_finger_print . key? ( url ) fp = @url_finger_print [ url ] else fp = response_body_md5 ( url ) end if @url_server . key? ( url ) server = @url_server [ url ] else server = get_server_header ( url ) end checker = Hash . new checker [ 'ip' ] = ip checker [ 'port' ] = port checker [ 'url' ] = url checker [ 'code' ] = code checker [ 'redirection' ] = loc checker [ 'md5' ] = fp checker [ 'server' ] = server checker [ 'timestamp' ] = timestamp if Wmap :: CidrTracker . new ( :data_dir => @data_dir ) . ip_trusted? ( ip ) checker [ 'status' ] = "int_hosted" else checker [ 'status' ] = "ext_hosted" end return checker rescue OpenSSL :: SSL :: SSLError => es checker = Hash . new checker [ 'ip' ] = ip checker [ 'port' ] = port checker [ 'url' ] = url checker [ 'code' ] = 20000 checker [ 'server' ] = "Unknown SSL error: #{es}" checker [ 'md' ] = nil checker [ 'redirection' ] = nil checker [ 'timestamp' ] = timestamp return checker end
Main worker method to perform various checks on the URL / site
Main worker method to perform various checks on the URL / site
# save the data # handler to temporally hold the openssl bug in bay:  SSL_set_session: unable to find ssl method #rescue Exception => ee #	puts "Exception on method #{__method__} for #{url}: #{ee}" # if @verbose #	return nil
url
<not_specific>