<reponame>ArtezGDA/MappingTheCity-Data
#!/usr/bin/python

# scrape_wiki_cities.py

import wikipedia
from bs4 import BeautifulSoup
import urllib
import json
from tqdm import tqdm

def appendLinkToCities(link, cityNameList, cityDictList):
	if link and link.has_attr('title'):
		cityName = link['title']
		# remove duplicates and broken pages
		if cityName and not cityName.endswith('(page does not exist)') and not cityName in cityNameList:
			# Check if this possible city has a coordinate
			try:
				p = wikipedia.page(cityName, auto_suggest=False)
				# Get the Coordinates 
				coord = p.coordinates
				cityDict = {}
				cityDict['name'] = cityName
				cityDict['lat'] = float(coord[0])
				cityDict['lon'] = float(coord[1])
				cityDict['url'] = p.url
				cityDictList.append(cityDict)
				cityNameList.append(cityName)
				return True
			except KeyError:
				return False
			except wikipedia.exceptions.DisambiguationError:
				return False
	return False


def main():
	"""Scrapes cities from a 'List of Cities in Country' page"""
	#
	# First get all the countries
	with open("countries.json", 'r') as inputFile:
	   citiesData = json.load(inputFile)
	print "Processing %d countries" % (len(citiesData))
	#
	# Open the partial json file (Or create an empty one)
	partialJsonFile = "cities_part.json"
	try:
		with open(partialJsonFile, 'r') as inputFile:
			partialData = json.load(inputFile)
	except IOError:
		partialData = []
	#
	# Then continue getting the cities
	numberOfCities = 0
	pBarCountries = tqdm(citiesData[:], leave=True, nested=True)
	for c in pBarCountries:
		# citiesList must not be an "" empty string
		country = c['country']
		listPage = c['citiesList']
		pBarCountries.set_description("Processed %s" % country)
		if listPage:
			p = wikipedia.page(listPage, auto_suggest=False)
			r = urllib.urlopen(p.url).read()
			soup = BeautifulSoup(r, "html.parser")
			tables = soup.findAll('table', class_="wikitable")
			# create two empty cities list
			# (one just for the city names, to prevent duplicates)
			# (the second to store more info per city)
			cities = []
			cityDicts = []
			possibleCityLinks = []
			if len(tables) > 0:
				# First the case when there are tables
				#
				for t in tables:
					for tr in t.findAll('tr'):
						allLinks = []
						for th in tr.findAll('th'):
							link = th.find('a')
							if link:
								allLinks.append(link)
						for td in tr.findAll('td'):
							link = td.find('a')
							if link:
								allLinks.append(link)
						possibleCityLinks.append(allLinks)		
			else:
				# Find all links in list items
				# First collect all possible container divs
				containerDivs = [soup.find('div', id="mw-content-text")]
				# Sub divs with multi-column lists
				containerDivs += soup.findAll('div', class_="div-col")
				# Search all divs
				for div in containerDivs:
					# Search all unordered lists
					for ul in div.findAll('ul', recursive=False):
						for link in ul.findAll('a'):
							possibleCityLinks.append([link])
					# Search all ordered lists		  
					for ol in div.findAll('ol', recursive=False):
						for link in ol.findAll('a'):
							possibleCityLinks.append([link])
			# Find first valid link
			pBarCities = tqdm(possibleCityLinks, leave=True, nested=True)
			for allLinks in pBarCities:
				# Find first valid link in the links
				for link in allLinks:
					if appendLinkToCities(link, cities, cityDicts):
						break
			numberOfCities += len(cityDicts)
			# Now, our cities list is filled with cities, but what to do with it.
			#
			# Let's append it to the dictionary c, we already had
			c['cities'] = cityDicts
			# Save to the partial json file
			partialData.append(c)
			with open(partialJsonFile, 'w') as outputFile:
				json.dump(partialData, outputFile, indent=2)
		else:
			# Add the empty country (with no citiesList) to the partial json
			partialData.append(c)
			with open(partialJsonFile, 'w') as outputFile:
				json.dump(partialData, outputFile, indent=2)
	# Save a file, with all cities by countries
	with open("cities.json", 'w') as outputFile:
	   json.dump(citiesData, outputFile, indent=2)
	print "Found list of %d cities" % (numberOfCities)
	

if __name__ == '__main__':
	main()
#!/usr/bin/python

# scrape_wiki_cities.py

import wikipedia
from bs4 import BeautifulSoup
import urllib
import json
from tqdm import tqdm

def appendLinkToCities(link, cityNameList, cityDictList):
	if link and link.has_attr('title'):
		cityName = link['title']
		# remove duplicates and broken pages
		if cityName and not cityName.endswith('(page does not exist)') and not cityName in cityNameList:
			# Check if this possible city has a coordinate
			try:
				p = wikipedia.page(cityName, auto_suggest=False)
				# Get the Coordinates 
				coord = p.coordinates
				cityDict = {}
				cityDict['name'] = cityName
				cityDict['lat'] = float(coord[0])
				cityDict['lon'] = float(coord[1])
				cityDict['url'] = p.url
				cityDictList.append(cityDict)
				cityNameList.append(cityName)
				return True
			except KeyError:
				return False
			except wikipedia.exceptions.DisambiguationError:
				return False
	return False


def main():
	"""Scrapes cities from a 'List of Cities in Country' page"""
	#
	# First get all the countries
	with open("countries.json", 'r') as inputFile:
	   citiesData = json.load(inputFile)
	print "Processing %d countries" % (len(citiesData))
	#
	# Open the partial json file (Or create an empty one)
	partialJsonFile = "cities_part.json"
	try:
		with open(partialJsonFile, 'r') as inputFile:
			partialData = json.load(inputFile)
	except IOError:
		partialData = []
	#
	# Then continue getting the cities
	numberOfCities = 0
	pBarCountries = tqdm(citiesData[:], leave=True, nested=True)
	for c in pBarCountries:
		# citiesList must not be an "" empty string
		country = c['country']
		listPage = c['citiesList']
		pBarCountries.set_description("Processed %s" % country)
		if listPage:
			p = wikipedia.page(listPage, auto_suggest=False)
			r = urllib.urlopen(p.url).read()
			soup = BeautifulSoup(r, "html.parser")
			tables = soup.findAll('table', class_="wikitable")
			# create two empty cities list
			# (one just for the city names, to prevent duplicates)
			# (the second to store more info per city)
			cities = []
			cityDicts = []
			possibleCityLinks = []
			if len(tables) > 0:
				# First the case when there are tables
				#
				for t in tables:
					for tr in t.findAll('tr'):
						allLinks = []
						for th in tr.findAll('th'):
							link = th.find('a')
							if link:
								allLinks.append(link)
						for td in tr.findAll('td'):
							link = td.find('a')
							if link:
								allLinks.append(link)
						possibleCityLinks.append(allLinks)		
			else:
				# Find all links in list items
				# First collect all possible container divs
				containerDivs = [soup.find('div', id="mw-content-text")]
				# Sub divs with multi-column lists
				containerDivs += soup.findAll('div', class_="div-col")
				# Search all divs
				for div in containerDivs:
					# Search all unordered lists
					for ul in div.findAll('ul', recursive=False):
						for link in ul.findAll('a'):
							possibleCityLinks.append([link])
					# Search all ordered lists		  
					for ol in div.findAll('ol', recursive=False):
						for link in ol.findAll('a'):
							possibleCityLinks.append([link])
			# Find first valid link
			pBarCities = tqdm(possibleCityLinks, leave=True, nested=True)
			for allLinks in pBarCities:
				# Find first valid link in the links
				for link in allLinks:
					if appendLinkToCities(link, cities, cityDicts):
						break
			numberOfCities += len(cityDicts)
			# Now, our cities list is filled with cities, but what to do with it.
			#
			# Let's append it to the dictionary c, we already had
			c['cities'] = cityDicts
			# Save to the partial json file
			partialData.append(c)
			with open(partialJsonFile, 'w') as outputFile:
				json.dump(partialData, outputFile, indent=2)
		else:
			# Add the empty country (with no citiesList) to the partial json
			partialData.append(c)
			with open(partialJsonFile, 'w') as outputFile:
				json.dump(partialData, outputFile, indent=2)
	# Save a file, with all cities by countries
	with open("cities.json", 'w') as outputFile:
	   json.dump(citiesData, outputFile, indent=2)
	print "Found list of %d cities" % (numberOfCities)
	

if __name__ == '__main__':
	main()
#!/usr/bin/python # scrape_wiki_cities.py # remove duplicates and broken pages # Check if this possible city has a coordinate # Get the Coordinates Scrapes cities from a 'List of Cities in Country' page # # First get all the countries # # Open the partial json file (Or create an empty one) # # Then continue getting the cities # citiesList must not be an "" empty string # create two empty cities list # (one just for the city names, to prevent duplicates) # (the second to store more info per city) # First the case when there are tables # # Find all links in list items # First collect all possible container divs # Sub divs with multi-column lists # Search all divs # Search all unordered lists # Search all ordered lists # Find first valid link # Find first valid link in the links # Now, our cities list is filled with cities, but what to do with it. # # Let's append it to the dictionary c, we already had # Save to the partial json file # Add the empty country (with no citiesList) to the partial json # Save a file, with all cities by countries