lib/wmap/site_tracker.rb
# Setter to add site entry to the cache one at a time
Setter to add site entry to the cache one at a time
Setter to add site entry to the cache one at a time
def add(site)
		puts "Add entry to the site store: #{site}"
		site=site.strip.downcase unless site.nil?
		if site_known?(site)
			puts  "Site already exists. Skip it: #{site}"
			return nil
		end
		site=normalize_url(site) if is_url?(site)
		site=url_2_site(site) if is_url?(site)
		puts "Site in standard format: #{site}" if @verbose
		raise "Exception on method #{__method__}: invalid site format of #{site}. Expected format is: http://your_website_name/" unless is_site?(site)
		trusted=false
		host=url_2_host(site)
		ip=host_2_ip(host)
		deact=Wmap::SiteTracker::DeactivatedSite.instance
		deact.sites_file=@data_dir + "/" + "deactivated_sites"
		File.write(deact.sites_file, "") unless File.exist?(deact.sites_file)
		deact.load_site_stores_from_file
		if is_ip?(host)
			trusted=Wmap::CidrTracker.new(:data_dir=>@data_dir).ip_trusted?(ip)
		else
			root=get_domain_root(host)
			if root.nil?
				raise "Invalid web site format. Please check your record again."
			else
				domain_tracker=Wmap::DomainTracker.instance
				domain_tracker.domains_file=@data_dir + "/" + "domains"
				File.write(domain_tracker.domains_file, "") unless File.exist?(domain_tracker.domains_file)
				domain_tracker.load_domains_from_file(domain_tracker.domains_file)
				trusted=domain_tracker.domain_known?(root)
				domain_tracker=nil
			end
		end
		host_tracker = Wmap::HostTracker.instance
		host_tracker.data_dir= @data_dir
		host_tracker.hosts_file = host_tracker.data_dir + "/" + "hosts"
		host_tracker.load_known_hosts_from_file(host_tracker.hosts_file)
		if trusted
			checker=Wmap::UrlChecker.new(:data_dir=>@data_dir).check(site)
			raise "Site is currently down. Skip #{site}" if checker.nil?
			raise "Site is time-out. Skip #{site}" if checker["code"] == 10000
			if is_https?(site)
			else
				raise "Site is currently down. Skip #{site}" if checker['code']==10000
			end
			raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip?(ip)
			if is_ip?(host)
				if host_tracker.ip_known?(host)
					puts "Local hosts table lookup for IP: #{ip}" if @verbose
					host=host_tracker.local_ip_2_host(host)
					puts "Host found from the local hosts table for #{ip}: #{host}" if @verbose
					site.sub!(/\d+\.\d+\.\d+\.\d+/,host)
				else
					puts "Reverse DNS lookup for IP: #{ip}" if @verbose
					host1=ip_2_host(host)
					puts "host1: #{host1}" if @verbose
					if is_fqdn?(host1)
						if host_tracker.domain_known?(host1)
							puts "Host found from the Internet reverse DNS lookup for #{ip}: #{host1}" if @verbose
							host=host1
							site.sub!(/\d+\.\d+\.\d+\.\d+/,host)
						end
					end
				end
				raise "Site already exist! Skip #{site}" if @known_sites.key?(site)
				puts "Adding site: #{site}" if @verbose
				@known_sites[site]=Hash.new
				@known_sites[site]=checker
				if deact.site_known?(site)
					deact.delete(site)
					deact.save!
				end
				puts "Site entry loaded: #{checker}"
				if is_fqdn?(host)
					puts "Update local hosts table for host: #{host}"
					if host_tracker.host_known?(host)
						old_ip=host_tracker.local_host_2_ip(host)
						if old_ip != ip
							host_tracker.refresh(host)
						else
							puts "Host resolve to the same IP #{ip} - no need to update the local host table." if @verbose
						end
					else
						host_tracker.add(host)
					end
				end
			else
				puts "Ading site: #{site}" if @verbose
				@known_sites[site]=Hash.new
				@known_sites[site]=checker
				if deact.site_known?(site)
					deact.delete(site)
					deact.save!
				end
				puts "Site entry loaded: #{checker}"
				puts "Update local hosts table for host: #{host}"
				if host_tracker.host_known?(host)
					old_ip=host_tracker.local_host_2_ip(host)
					if old_ip != ip
						host_tracker.refresh(host)
					else
					end
				else
					host_tracker.add(host)
				end
			end
			deact=nil
			host_tracker=nil
			return checker
		else
			puts "Problem found: untrusted Internet domain or IP. Skip #{site}"
			deact=nil
			host_tracker.save!
			host_tracker=nil
			return nil
		end
	rescue => ee
		puts "Exception on method #{__method__}: #{ee}"
		checker=nil
		deact=nil
		host_tracker=nil
		return nil
	end
def add ( site ) puts "Add entry to the site store: #{site}" site = site . strip . downcase unless site . nil? if site_known? ( site ) puts "Site already exists. Skip it: #{site}" return nil end site = normalize_url ( site ) if is_url? ( site ) site = url_2_site ( site ) if is_url? ( site ) puts "Site in standard format: #{site}" if @verbose raise "Exception on method #{__method__}: invalid site format of #{site}. Expected format is: http://your_website_name/" unless is_site? ( site ) trusted = false host = url_2_host ( site ) ip = host_2_ip ( host ) deact = Wmap :: SiteTracker :: DeactivatedSite . instance deact . sites_file = @data_dir + "/" + "deactivated_sites" File . write ( deact . sites_file , "" ) unless File . exist? ( deact . sites_file ) deact . load_site_stores_from_file if is_ip? ( host ) trusted = Wmap :: CidrTracker . new ( :data_dir => @data_dir ) . ip_trusted? ( ip ) else root = get_domain_root ( host ) if root . nil? raise "Invalid web site format. Please check your record again." else domain_tracker = Wmap :: DomainTracker . instance domain_tracker . domains_file = @data_dir + "/" + "domains" File . write ( domain_tracker . domains_file , "" ) unless File . exist? ( domain_tracker . domains_file ) domain_tracker . load_domains_from_file ( domain_tracker . domains_file ) trusted = domain_tracker . domain_known? ( root ) domain_tracker = nil end end host_tracker = Wmap :: HostTracker . instance host_tracker . data_dir = @data_dir host_tracker . hosts_file = host_tracker . data_dir + "/" + "hosts" host_tracker . load_known_hosts_from_file ( host_tracker . hosts_file ) if trusted checker = Wmap :: UrlChecker . new ( :data_dir => @data_dir ) . check ( site ) raise "Site is currently down. Skip #{site}" if checker . nil? raise "Site is time-out. Skip #{site}" if checker [ "code" ] == 10000 if is_https? ( site ) else raise "Site is currently down. Skip #{site}" if checker [ 'code' ] == 10000 end raise "Exception on add method - Fail to resolve the host-name: Host - #{host}, IP - #{ip}. Skip #{site}" unless is_ip? ( ip ) if is_ip? ( host ) if host_tracker . ip_known? ( host ) puts "Local hosts table lookup for IP: #{ip}" if @verbose host = host_tracker . local_ip_2_host ( host ) puts "Host found from the local hosts table for #{ip}: #{host}" if @verbose site . sub! ( / \d + \. \d + \. \d + \. \d + / , host ) else puts "Reverse DNS lookup for IP: #{ip}" if @verbose host1 = ip_2_host ( host ) puts "host1: #{host1}" if @verbose if is_fqdn? ( host1 ) if host_tracker . domain_known? ( host1 ) puts "Host found from the Internet reverse DNS lookup for #{ip}: #{host1}" if @verbose host = host1 site . sub! ( / \d + \. \d + \. \d + \. \d + / , host ) end end end raise "Site already exist! Skip #{site}" if @known_sites . key? ( site ) puts "Adding site: #{site}" if @verbose @known_sites [ site ] = Hash . new @known_sites [ site ] = checker if deact . site_known? ( site ) deact . delete ( site ) deact . save! end puts "Site entry loaded: #{checker}" if is_fqdn? ( host ) puts "Update local hosts table for host: #{host}" if host_tracker . host_known? ( host ) old_ip = host_tracker . local_host_2_ip ( host ) if old_ip != ip host_tracker . refresh ( host ) else puts "Host resolve to the same IP #{ip} - no need to update the local host table." if @verbose end else host_tracker . add ( host ) end end else puts "Ading site: #{site}" if @verbose @known_sites [ site ] = Hash . new @known_sites [ site ] = checker if deact . site_known? ( site ) deact . delete ( site ) deact . save! end puts "Site entry loaded: #{checker}" puts "Update local hosts table for host: #{host}" if host_tracker . host_known? ( host ) old_ip = host_tracker . local_host_2_ip ( host ) if old_ip != ip host_tracker . refresh ( host ) else end else host_tracker . add ( host ) end end deact = nil host_tracker = nil return checker else puts "Problem found: untrusted Internet domain or IP. Skip #{site}" deact = nil host_tracker . save! host_tracker = nil return nil end rescue => ee puts "Exception on method #{__method__}: #{ee}" checker = nil deact = nil host_tracker = nil return nil end
Setter to add site entry to the cache one at a time
Setter to add site entry to the cache one at a time
# Preliminary sanity check # Additional logic to refresh deactivated site, 02/12/2014 # only trust either the domain or IP we know # add record only if trusted # Add logic to check site status before adding it # Skip the http site if it's un-responsive; for the https we'll keep it because we're interested in analysing the SSL layer later # do nothing # Update the local host table when necessary # Case #1: Trusted site contains IP # Try local reverse DNS lookup first # Try reverse DNS lookup over Internet as secondary precaution # replace IP with host-name only if domain root is known # Adding site for Case #1 # Add logic to update the hosts table for case #1 variance # -  case that reverse DNS lookup successful # Case #2: Trusted site contains valid FQDN # Add logic to update the hosts table for case #2 # Skip - no need to update the local hosts table
site
<not_specific>