scraper/scrape-to-csv.rb
# wrapper around a cache a webpages (name id md5 hash of url, contents are raw HTML)
wrapper around a cache a webpages (name id md5 hash of url, contents are raw HTML)
wrapper around a cache a webpages ( name id md5 hash of url contents are raw HTML )
def cached_wget(url, override_cache=false)
	puts "  Fetching #{url}"
	cache_file_name = File.join CACHE_DIR, Digest::MD5.hexdigest(url)
	if override_cache or not File.exists? cache_file_name
		File.open(cache_file_name, 'w') do |cache_file|  
			open(url) do |remote_file|
				remote_file.each_line do |remote_line| 
					cache_file.puts remote_line
				end
			end
		end 
		puts " (downloaded fresh copy to #{cache_file_name})"
	else
		puts " (loaded from cache #{cache_file_name})"
	end
	cache_file_name
end
def cached_wget ( url , override_cache = false ) puts "  Fetching #{url}" cache_file_name = File . join CACHE_DIR , Digest :: MD5 . hexdigest ( url ) if override_cache or not File . exists? cache_file_name File . open ( cache_file_name , 'w' ) do | cache_file | open ( url ) do | remote_file | remote_file . each_line do | remote_line | cache_file . puts remote_line end end end puts " (downloaded fresh copy to #{cache_file_name})" else puts " (loaded from cache #{cache_file_name})" end cache_file_name end
wrapper around a cache a webpages (name id md5 hash of url, contents are raw HTML)
wrapper around a cache a webpages ( name id md5 hash of url contents are raw HTML )
url
override_cache