#--  --  --  -- Python Data Science Toolbox (Part 2):
# Used for Data Scientist Training Path 
#FYI it's a compilation of how to work
#with different commands.


### --------------------------------------------------------
# # ------>>>>>Iterators vs Iterables
# Let's do a quick recall of what you've learned 
# about iterables and iterators. Recall from the video 
# that an iterable is an object that can return an iterator,
#  while an iterator is an object that keeps state and produces 
# the next value when you call next() on it. In this exercise, 
# you will identify which object is an iterable and which is an iterator.
# The environment has been pre-loaded with the variables flash1 and flash2. 
# Try printing out their values with print() and next() to figure out which 
# is an iterable and which is an iterator
# R/ flash1 is an iterable and flash2 is an iterator.



### --------------------------------------------------------
# # ------>>>>> Iterating over iterables - ex#0
# Create a list of strings: flash
flash = ['<NAME>', '<NAME>', 'wally west', 'bart allen']
# Print each list item in flash using a for loop
for name in flash:
    print(name)
# Create an iterator for flash: superspeed
superhero = iter(flash)
# Print each item from the iterator
print(next(superhero))
print(next(superhero))
print(next(superhero))
print(next(superhero))


### --------------------------------------------------------
# # ------>>>>> Iterating over iterables - ex#1
# Create an iterator for range(3): small_value
small_value = iter(range(3))
# Print the values in small_value
print(next(small_value))
print(next(small_value))
print(next(small_value))
# Loop over range(3) and print the values
for num in range(3):
    print(num)
# Create an iterator for range(10 ** 100): googol
googol = iter(range(10 ** 100))
# Print the first 5 values from googol
print(next(googol))
print(next(googol))
print(next(googol))
print(next(googol))
print(next(googol))

### --------------------------------------------------------
# # ------>>>>> Iterators as function arguments
# Create a range object: values
values = range(10, 21)
# Print the range object
print(values)
# Create a list of integers: values_list
values_list = list(values)
# Print values_list
print(values_list)
# Get the sum of values: values_sum
values_sum = sum(values)
# Print values_sum
print(values_sum)


### --------------------------------------------------------
# Create a list of strings: mutants
mutants = ['<NAME>', 
            '<NAME>', 
            '<NAME>', 
            '<NAME>', 
            '<NAME>']
# Create a list of tuples: mutant_list
mutant_list = list(enumerate(mutants))
# Print the list of tuples
print(mutant_list)
# Unpack and print the tuple pairs
for index1, value1 in enumerate(mutants):
    print(index1, value1)
# Change the start index
for index2, value2 in enumerate(mutants, start=1):
    print(index2, value2)

### --------------------------------------------------------
# # ------>>>>> Using zip
# Create a list of tuples: mutant_data
mutant_data = list(zip(mutants, aliases, powers))
# Print the list of tuples
print(mutant_data)
# Create a zip object using the three lists: mutant_zip
mutant_zip = zip(mutants, aliases, powers)
# Print the zip object
print(mutant_zip)
# Unpack the zip object and print the tuple values
# Unpack the zip object and print the tuple values
for value1, value2, value3 in mutant_zip:
    print(value1, value2, value3)

### --------------------------------------------------------
# # ------>>>>> Using * and zip to 'unzip'
# Create a zip object from mutants and powers: z1
z1 = zip(mutants, powers)
# Print the tuples in z1 by unpacking with *
print(*z1)
# Re-create a zip object from mutants and powers: z1
z1 = zip(mutants, powers)
# 'Unzip' the tuples in z1 by unpacking with * and zip(): result1, result2
result1, result2 = zip(*z1)
# Check if unpacked tuples are equivalent to original tuples
print(result1 == mutants)
print(result2 == powers)

### --------------------------------------------------------
# # ------>>>>> Processing large amounts of Twitter data
# Initialize an empty dictionary: counts_dict
counts_dict = {}
# Iterate over the file chunk by chunk
for chunk in pd.read_csv('./tweets.csv', chunksize=10):
    # Iterate over the column in DataFrame
    for entry in chunk['lang']:
        if entry in counts_dict.keys():
            counts_dict[entry] += 1
        else:
            counts_dict[entry] = 1
# Print the populated dictionary
print(counts_dict)



### --------------------------------------------------------
# # ------>>>>> Extracting information for large amounts of Twitter data
# Define count_entries()
def count_entries(csv_file, c_size, colname):
    """Return a dictionary with counts of
    occurrences as value for each key."""
    # Initialize an empty dictionary: counts_dict
    counts_dict = {}
    # Iterate over the file chunk by chunk
    for chunk in pd.read_csv(csv_file, chunksize=c_size):
        # Iterate over the column in DataFrame
        for entry in chunk[colname]:
            if entry in counts_dict.keys():
                counts_dict[entry] += 1
            else:
                counts_dict[entry] = 1
    # Return counts_dict
    return counts_dict
# Call count_entries(): result_counts
result_counts = count_entries('./tweets.csv', 10, 'lang')
# Print result_counts
print(result_counts)


### --------------------------------------------------------
# # ------>>>>> Write a basic list comprehension
# In this exercise, you will practice what you've learned from 
# the video about writing list comprehensions. You will write a 
# list comprehension and identify the output that will be produced.
# The following list has been pre-loaded in the environment.
# doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson']
# How would a list comprehension that produces a list of the first 
# character of each string in doctor look like? Note that the list 
# comprehension uses doc as the iterator variable. What will the output be?
# R/ The list comprehension is [doc[0] for doc in doctor] and produces 
# the list ['h', 'c', 'c', 't', 'w'].


### --------------------------------------------------------
# # ------>>>>> List comprehension over iterables
# You know that list comprehensions can be built over iterables. 
# Given the following objects below, which of these can we build list comprehensions over?
# doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson']
# range(50)
# underwood = 'After all, we are nothing more or less than what we choose to reveal.'
# jean = '24601'
# flash = ['<NAME>', '<NAME>', 'w<NAME>', '<NAME>']
# valjean = 24601
# R/ You can build list comprehensions over all the objects except the integer object valjean.


### --------------------------------------------------------
# # ------>>>>> Writing list comprehensions
# Create list comprehension: squares
squares = [i ** 2 for i in range(10)]
print(squares)


### --------------------------------------------------------
# # ------>>>>> Nested list comprehensions
# Create a 5 x 5 matrix using a list of lists: matrix
matrix = [[col for col in range(5)] for row in range(5)]
# Print the matrix
for row in matrix:
    print(row)


### --------------------------------------------------------
# # ------>>>>> Using conditionals in comprehensions - ex#0
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create list comprehension: new_fellowship
new_fellowship = [member for member in fellowship if len(member) >= 7]
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> Using conditionals in comprehensions - ex#1
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create list comprehension: new_fellowship
new_fellowship = [member if len(member) >= 7 else member.replace(
    member, '') for member in fellowship]
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> Dict comprehensions
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create dict comprehension: new_fellowship
new_fellowship = {member: len(member) for member in fellowship}
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> List comprehensions vs generators
# You've seen from the videos that list comprehensions and generator 
# expressions look very similar in their syntax, except for the use of
#  parentheses () in generator expressions and brackets [] in list comprehensions.
# In this exercise, you will recall the difference between
# list comprehensions and generators. To help with that task, 
# the following code has been pre-loaded in the environment:
# # List of strings
# fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# # List comprehension
# fellow1 = [member for member in fellowship if len(member) >= 7]
# # Generator expression
# fellow2 = (member for member in fellowship if len(member) >= 7)
# Try to play around with fellow1 and fellow2 by figuring out their types and 
# printing out their values. Based on your observations and what you can recall
# from the video, select from the options below the best description for the 
# difference between list comprehensions and generators.
# R/ A list comprehension produces a list as output, a generator produces a generator object.


### --------------------------------------------------------
# # ------>>>>> Write your own generator expressions
# Create generator object: result
result = (num for num in range(31))
# Print the first 5 values
print(next(result))
print(next(result))
print(next(result))
print(next(result))
print(next(result))
# Print the rest of the values
for value in result:
    print(value)


### --------------------------------------------------------
# # ------>>>>> Changing the output in generator expressions
# Create a list of strings: lannister
lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']
# Create a generator object: lengths
lengths = (len(person) for person in lannister)
# Iterate over and print the values in lengths
for value in lengths:
    print(value)


### --------------------------------------------------------
# # ------>>>>> Build a generator
# Create a list of strings
lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']
# Define generator function get_lengths
def get_lengths(input_list):
    """Generator function that yields the
    length of the strings in input_list."""
    # Yield the length of a string
    for person in input_list:
        yield len(person)
# Print the values generated by get_lengths()
for value in get_lengths(lannister):
    print(value)


### --------------------------------------------------------
# # ------>>>>> List comprehensions for time-stamped data
# Extract the created_at column from df: tweet_time
tweet_time = df['created_at']
# Extract the clock time: tweet_clock_time
tweet_clock_time = [entry[11:19] for entry in tweet_time]
# Print the extracted times
print(tweet_clock_time)


### --------------------------------------------------------
# # ------>>>>> Conditional list comprehensions for time-stamped data
# Extract the created_at column from df: tweet_time
tweet_time = df['created_at']
# Extract the clock time: tweet_clock_time
tweet_clock_time = [entry[11:19] for entry in tweet_time if entry[17:19] == '19']
# Print the extracted times
print(tweet_clock_time)


### --------------------------------------------------------
# # ------>>>>> Dictionaries for data science
# Zip lists: zipped_lists
zipped_lists = zip(feature_names, row_vals)
# Create a dictionary: rs_dict
rs_dict = dict(zipped_lists)
# Print the dictionary
print(rs_dict)

 

### --------------------------------------------------------
# # ------>>>>> Writing a function to help you
# Define lists2dict()
def lists2dict(list1, list2):
    """Return a dictionary where list1 provides
    the keys and list2 provides the values."""
    # Zip lists: zipped_lists
    zipped_lists = zip(list1, list2)
    # Create a dictionary: rs_dict
    rs_dict = dict(zipped_lists)
    # Return the dictionary
    return rs_dict
# Call lists2dict: rs_fxn
rs_fxn = lists2dict(feature_names, row_vals)
# Print rs_fxn
print(rs_fxn)


### --------------------------------------------------------
# # ------>>>>> Using a list comprehension
# Print the first two lists in row_lists
print(row_lists[0])
print(row_lists[1])
# Turn list of lists into list of dicts: list_of_dicts
list_of_dicts = [lists2dict(feature_names, sublist) for sublist in row_lists]
# Print the first two dictionaries in list_of_dicts
print(list_of_dicts[0])
print(list_of_dicts[1])



### --------------------------------------------------------
# # ------>>>>> Turning this all into a DataFrame
# Import the pandas package
import pandas as pd
# Turn list of lists into list of dicts: list_of_dicts
list_of_dicts = [lists2dict(feature_names, sublist) for sublist in row_lists]
# Turn list of dicts into a DataFrame: df
df = pd.DataFrame(list_of_dicts)
# Print the head of the DataFrame
print(df.head())


### --------------------------------------------------------
# # ------>>>>> Processing data in chunks - ex#0
with open('world_dev_ind.csv') as file:
    # Skip the column names
    file.readline()
    # Initialize an empty dictionary: counts_dict
    counts_dict = {}
    # Process only the first 1000 rows
    for j in range(0, 1000):
        # Split the current line into a list: line
        line = file.readline().split(',')
        # Get the value for the first column: first_col
        first_col = line[0]
        # If the column value is in the dict, increment its value
        if first_col in counts_dict.keys():
            counts_dict[first_col] += 1

        # Else, add to the dict and set value to 1
        else:
            counts_dict[first_col] = 1
# Print the resulting dictionary
print(counts_dict)

### --------------------------------------------------------
# # ------>>>>> Writing a generator to load data in chunks (2)
# Define read_large_file()
def read_large_file(file_object):
    """A generator function to read a large file lazily."""
    # Loop indefinitely until the end of the file
    while True:

        # Read a line from the file: data
        data = file_object.readline()
        # Break if this is the end of the file
        if not data:
            break
        # Yield the line of data
        yield data
# Open a connection to the file
with open('world_dev_ind.csv') as file:
    # Create a generator object for the file: gen_file
    gen_file = read_large_file(file)
    # Print the first three lines of the file
    print(next(gen_file))
    print(next(gen_file))
    print(next(gen_file))


### --------------------------------------------------------
# # ------>>>>> Writing a generator to load data in chunks (3)
# Initialize an empty dictionary: counts_dict
counts_dict = {}
# Open a connection to the file
with open('world_dev_ind.csv') as file:
    # Iterate over the generator from read_large_file()
    for line in read_large_file(file):
        row = line.split(',')
        first_col = row[0]
        if first_col in counts_dict.keys():
            counts_dict[first_col] += 1
        else:
            counts_dict[first_col] = 1
# Print            
print(counts_dict)


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#0
# Import the pandas package
import pandas as pd
# Initialize reader object: df_reader
df_reader = pd.read_csv('ind_pop.csv', chunksize=10)
# Print two chunks
print(next(df_reader))
print(next(df_reader))


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#1
# Initialize reader object: urb_pop_reader
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
# Get the first DataFrame chunk: df_urb_pop
df_urb_pop = next(urb_pop_reader)
# Check out the head of the DataFrame
print(df_urb_pop.head())
# Check out specific country: df_pop_ceb
df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
# Zip DataFrame columns of interest: pops
pops = zip(df_pop_ceb['Total Population'], 
            df_pop_ceb['Urban population (% of total)'])
# Turn zip object into list: pops_list
pops_list = list(pops)
# Print pops_list
print(pops_list)


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#2
# Code from previous exercise
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
df_urb_pop = next(urb_pop_reader)
df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
pops = zip(df_pop_ceb['Total Population'], 
           df_pop_ceb['Urban population (% of total)'])
pops_list = list(pops)
# Use list comprehension to create new DataFrame column 'Total Urban Population'
df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]
# Plot urban population data
df_pop_ceb.plot(kind='scatter', x='Year', y='Total Urban Population')
plt.show()

### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#3
# Initialize reader object: urb_pop_reader
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
# Initialize empty DataFrame: data
data = pd.DataFrame()
# Iterate over each DataFrame chunk
for df_urb_pop in urb_pop_reader:
    # Check out specific country: df_pop_ceb
    df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
    # Zip DataFrame columns of interest: pops
    pops = zip(df_pop_ceb['Total Population'],
                df_pop_ceb['Urban population (% of total)'])
    # Turn zip object into list: pops_list
    pops_list = list(pops)
    # Use list comprehension to create new DataFrame column 'Total Urban Population'
    df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]
    # Append DataFrame chunk to data: data
    data = data.append(df_pop_ceb)
# Plot urban population data
data.plot(kind='scatter', x='Year', y='Total Urban Population')
plt.show()


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#4
#--  --  --  -- Python Data Science Toolbox (Part 2):
# Used for Data Scientist Training Path 
#FYI it's a compilation of how to work
#with different commands.


### --------------------------------------------------------
# # ------>>>>>Iterators vs Iterables
# Let's do a quick recall of what you've learned 
# about iterables and iterators. Recall from the video 
# that an iterable is an object that can return an iterator,
#  while an iterator is an object that keeps state and produces 
# the next value when you call next() on it. In this exercise, 
# you will identify which object is an iterable and which is an iterator.
# The environment has been pre-loaded with the variables flash1 and flash2. 
# Try printing out their values with print() and next() to figure out which 
# is an iterable and which is an iterator
# R/ flash1 is an iterable and flash2 is an iterator.



### --------------------------------------------------------
# # ------>>>>> Iterating over iterables - ex#0
# Create a list of strings: flash
flash = ['<NAME>', '<NAME>', 'wally west', 'bart allen']
# Print each list item in flash using a for loop
for name in flash:
    print(name)
# Create an iterator for flash: superspeed
superhero = iter(flash)
# Print each item from the iterator
print(next(superhero))
print(next(superhero))
print(next(superhero))
print(next(superhero))


### --------------------------------------------------------
# # ------>>>>> Iterating over iterables - ex#1
# Create an iterator for range(3): small_value
small_value = iter(range(3))
# Print the values in small_value
print(next(small_value))
print(next(small_value))
print(next(small_value))
# Loop over range(3) and print the values
for num in range(3):
    print(num)
# Create an iterator for range(10 ** 100): googol
googol = iter(range(10 ** 100))
# Print the first 5 values from googol
print(next(googol))
print(next(googol))
print(next(googol))
print(next(googol))
print(next(googol))

### --------------------------------------------------------
# # ------>>>>> Iterators as function arguments
# Create a range object: values
values = range(10, 21)
# Print the range object
print(values)
# Create a list of integers: values_list
values_list = list(values)
# Print values_list
print(values_list)
# Get the sum of values: values_sum
values_sum = sum(values)
# Print values_sum
print(values_sum)


### --------------------------------------------------------
# Create a list of strings: mutants
mutants = ['<NAME>', 
            '<NAME>', 
            '<NAME>', 
            '<NAME>', 
            '<NAME>']
# Create a list of tuples: mutant_list
mutant_list = list(enumerate(mutants))
# Print the list of tuples
print(mutant_list)
# Unpack and print the tuple pairs
for index1, value1 in enumerate(mutants):
    print(index1, value1)
# Change the start index
for index2, value2 in enumerate(mutants, start=1):
    print(index2, value2)

### --------------------------------------------------------
# # ------>>>>> Using zip
# Create a list of tuples: mutant_data
mutant_data = list(zip(mutants, aliases, powers))
# Print the list of tuples
print(mutant_data)
# Create a zip object using the three lists: mutant_zip
mutant_zip = zip(mutants, aliases, powers)
# Print the zip object
print(mutant_zip)
# Unpack the zip object and print the tuple values
# Unpack the zip object and print the tuple values
for value1, value2, value3 in mutant_zip:
    print(value1, value2, value3)

### --------------------------------------------------------
# # ------>>>>> Using * and zip to 'unzip'
# Create a zip object from mutants and powers: z1
z1 = zip(mutants, powers)
# Print the tuples in z1 by unpacking with *
print(*z1)
# Re-create a zip object from mutants and powers: z1
z1 = zip(mutants, powers)
# 'Unzip' the tuples in z1 by unpacking with * and zip(): result1, result2
result1, result2 = zip(*z1)
# Check if unpacked tuples are equivalent to original tuples
print(result1 == mutants)
print(result2 == powers)

### --------------------------------------------------------
# # ------>>>>> Processing large amounts of Twitter data
# Initialize an empty dictionary: counts_dict
counts_dict = {}
# Iterate over the file chunk by chunk
for chunk in pd.read_csv('./tweets.csv', chunksize=10):
    # Iterate over the column in DataFrame
    for entry in chunk['lang']:
        if entry in counts_dict.keys():
            counts_dict[entry] += 1
        else:
            counts_dict[entry] = 1
# Print the populated dictionary
print(counts_dict)



### --------------------------------------------------------
# # ------>>>>> Extracting information for large amounts of Twitter data
# Define count_entries()
def count_entries(csv_file, c_size, colname):
    """Return a dictionary with counts of
    occurrences as value for each key."""
    # Initialize an empty dictionary: counts_dict
    counts_dict = {}
    # Iterate over the file chunk by chunk
    for chunk in pd.read_csv(csv_file, chunksize=c_size):
        # Iterate over the column in DataFrame
        for entry in chunk[colname]:
            if entry in counts_dict.keys():
                counts_dict[entry] += 1
            else:
                counts_dict[entry] = 1
    # Return counts_dict
    return counts_dict
# Call count_entries(): result_counts
result_counts = count_entries('./tweets.csv', 10, 'lang')
# Print result_counts
print(result_counts)


### --------------------------------------------------------
# # ------>>>>> Write a basic list comprehension
# In this exercise, you will practice what you've learned from 
# the video about writing list comprehensions. You will write a 
# list comprehension and identify the output that will be produced.
# The following list has been pre-loaded in the environment.
# doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson']
# How would a list comprehension that produces a list of the first 
# character of each string in doctor look like? Note that the list 
# comprehension uses doc as the iterator variable. What will the output be?
# R/ The list comprehension is [doc[0] for doc in doctor] and produces 
# the list ['h', 'c', 'c', 't', 'w'].


### --------------------------------------------------------
# # ------>>>>> List comprehension over iterables
# You know that list comprehensions can be built over iterables. 
# Given the following objects below, which of these can we build list comprehensions over?
# doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson']
# range(50)
# underwood = 'After all, we are nothing more or less than what we choose to reveal.'
# jean = '24601'
# flash = ['<NAME>', '<NAME>', 'w<NAME>', '<NAME>']
# valjean = 24601
# R/ You can build list comprehensions over all the objects except the integer object valjean.


### --------------------------------------------------------
# # ------>>>>> Writing list comprehensions
# Create list comprehension: squares
squares = [i ** 2 for i in range(10)]
print(squares)


### --------------------------------------------------------
# # ------>>>>> Nested list comprehensions
# Create a 5 x 5 matrix using a list of lists: matrix
matrix = [[col for col in range(5)] for row in range(5)]
# Print the matrix
for row in matrix:
    print(row)


### --------------------------------------------------------
# # ------>>>>> Using conditionals in comprehensions - ex#0
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create list comprehension: new_fellowship
new_fellowship = [member for member in fellowship if len(member) >= 7]
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> Using conditionals in comprehensions - ex#1
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create list comprehension: new_fellowship
new_fellowship = [member if len(member) >= 7 else member.replace(
    member, '') for member in fellowship]
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> Dict comprehensions
# Create a list of strings: fellowship
fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# Create dict comprehension: new_fellowship
new_fellowship = {member: len(member) for member in fellowship}
# Print the new list
print(new_fellowship)


### --------------------------------------------------------
# # ------>>>>> List comprehensions vs generators
# You've seen from the videos that list comprehensions and generator 
# expressions look very similar in their syntax, except for the use of
#  parentheses () in generator expressions and brackets [] in list comprehensions.
# In this exercise, you will recall the difference between
# list comprehensions and generators. To help with that task, 
# the following code has been pre-loaded in the environment:
# # List of strings
# fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli']
# # List comprehension
# fellow1 = [member for member in fellowship if len(member) >= 7]
# # Generator expression
# fellow2 = (member for member in fellowship if len(member) >= 7)
# Try to play around with fellow1 and fellow2 by figuring out their types and 
# printing out their values. Based on your observations and what you can recall
# from the video, select from the options below the best description for the 
# difference between list comprehensions and generators.
# R/ A list comprehension produces a list as output, a generator produces a generator object.


### --------------------------------------------------------
# # ------>>>>> Write your own generator expressions
# Create generator object: result
result = (num for num in range(31))
# Print the first 5 values
print(next(result))
print(next(result))
print(next(result))
print(next(result))
print(next(result))
# Print the rest of the values
for value in result:
    print(value)


### --------------------------------------------------------
# # ------>>>>> Changing the output in generator expressions
# Create a list of strings: lannister
lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']
# Create a generator object: lengths
lengths = (len(person) for person in lannister)
# Iterate over and print the values in lengths
for value in lengths:
    print(value)


### --------------------------------------------------------
# # ------>>>>> Build a generator
# Create a list of strings
lannister = ['cersei', 'jaime', 'tywin', 'tyrion', 'joffrey']
# Define generator function get_lengths
def get_lengths(input_list):
    """Generator function that yields the
    length of the strings in input_list."""
    # Yield the length of a string
    for person in input_list:
        yield len(person)
# Print the values generated by get_lengths()
for value in get_lengths(lannister):
    print(value)


### --------------------------------------------------------
# # ------>>>>> List comprehensions for time-stamped data
# Extract the created_at column from df: tweet_time
tweet_time = df['created_at']
# Extract the clock time: tweet_clock_time
tweet_clock_time = [entry[11:19] for entry in tweet_time]
# Print the extracted times
print(tweet_clock_time)


### --------------------------------------------------------
# # ------>>>>> Conditional list comprehensions for time-stamped data
# Extract the created_at column from df: tweet_time
tweet_time = df['created_at']
# Extract the clock time: tweet_clock_time
tweet_clock_time = [entry[11:19] for entry in tweet_time if entry[17:19] == '19']
# Print the extracted times
print(tweet_clock_time)


### --------------------------------------------------------
# # ------>>>>> Dictionaries for data science
# Zip lists: zipped_lists
zipped_lists = zip(feature_names, row_vals)
# Create a dictionary: rs_dict
rs_dict = dict(zipped_lists)
# Print the dictionary
print(rs_dict)

 

### --------------------------------------------------------
# # ------>>>>> Writing a function to help you
# Define lists2dict()
def lists2dict(list1, list2):
    """Return a dictionary where list1 provides
    the keys and list2 provides the values."""
    # Zip lists: zipped_lists
    zipped_lists = zip(list1, list2)
    # Create a dictionary: rs_dict
    rs_dict = dict(zipped_lists)
    # Return the dictionary
    return rs_dict
# Call lists2dict: rs_fxn
rs_fxn = lists2dict(feature_names, row_vals)
# Print rs_fxn
print(rs_fxn)


### --------------------------------------------------------
# # ------>>>>> Using a list comprehension
# Print the first two lists in row_lists
print(row_lists[0])
print(row_lists[1])
# Turn list of lists into list of dicts: list_of_dicts
list_of_dicts = [lists2dict(feature_names, sublist) for sublist in row_lists]
# Print the first two dictionaries in list_of_dicts
print(list_of_dicts[0])
print(list_of_dicts[1])



### --------------------------------------------------------
# # ------>>>>> Turning this all into a DataFrame
# Import the pandas package
import pandas as pd
# Turn list of lists into list of dicts: list_of_dicts
list_of_dicts = [lists2dict(feature_names, sublist) for sublist in row_lists]
# Turn list of dicts into a DataFrame: df
df = pd.DataFrame(list_of_dicts)
# Print the head of the DataFrame
print(df.head())


### --------------------------------------------------------
# # ------>>>>> Processing data in chunks - ex#0
with open('world_dev_ind.csv') as file:
    # Skip the column names
    file.readline()
    # Initialize an empty dictionary: counts_dict
    counts_dict = {}
    # Process only the first 1000 rows
    for j in range(0, 1000):
        # Split the current line into a list: line
        line = file.readline().split(',')
        # Get the value for the first column: first_col
        first_col = line[0]
        # If the column value is in the dict, increment its value
        if first_col in counts_dict.keys():
            counts_dict[first_col] += 1

        # Else, add to the dict and set value to 1
        else:
            counts_dict[first_col] = 1
# Print the resulting dictionary
print(counts_dict)

### --------------------------------------------------------
# # ------>>>>> Writing a generator to load data in chunks (2)
# Define read_large_file()
def read_large_file(file_object):
    """A generator function to read a large file lazily."""
    # Loop indefinitely until the end of the file
    while True:

        # Read a line from the file: data
        data = file_object.readline()
        # Break if this is the end of the file
        if not data:
            break
        # Yield the line of data
        yield data
# Open a connection to the file
with open('world_dev_ind.csv') as file:
    # Create a generator object for the file: gen_file
    gen_file = read_large_file(file)
    # Print the first three lines of the file
    print(next(gen_file))
    print(next(gen_file))
    print(next(gen_file))


### --------------------------------------------------------
# # ------>>>>> Writing a generator to load data in chunks (3)
# Initialize an empty dictionary: counts_dict
counts_dict = {}
# Open a connection to the file
with open('world_dev_ind.csv') as file:
    # Iterate over the generator from read_large_file()
    for line in read_large_file(file):
        row = line.split(',')
        first_col = row[0]
        if first_col in counts_dict.keys():
            counts_dict[first_col] += 1
        else:
            counts_dict[first_col] = 1
# Print            
print(counts_dict)


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#0
# Import the pandas package
import pandas as pd
# Initialize reader object: df_reader
df_reader = pd.read_csv('ind_pop.csv', chunksize=10)
# Print two chunks
print(next(df_reader))
print(next(df_reader))


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#1
# Initialize reader object: urb_pop_reader
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
# Get the first DataFrame chunk: df_urb_pop
df_urb_pop = next(urb_pop_reader)
# Check out the head of the DataFrame
print(df_urb_pop.head())
# Check out specific country: df_pop_ceb
df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
# Zip DataFrame columns of interest: pops
pops = zip(df_pop_ceb['Total Population'], 
            df_pop_ceb['Urban population (% of total)'])
# Turn zip object into list: pops_list
pops_list = list(pops)
# Print pops_list
print(pops_list)


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#2
# Code from previous exercise
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
df_urb_pop = next(urb_pop_reader)
df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
pops = zip(df_pop_ceb['Total Population'], 
           df_pop_ceb['Urban population (% of total)'])
pops_list = list(pops)
# Use list comprehension to create new DataFrame column 'Total Urban Population'
df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]
# Plot urban population data
df_pop_ceb.plot(kind='scatter', x='Year', y='Total Urban Population')
plt.show()

### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#3
# Initialize reader object: urb_pop_reader
urb_pop_reader = pd.read_csv('ind_pop_data.csv', chunksize=1000)
# Initialize empty DataFrame: data
data = pd.DataFrame()
# Iterate over each DataFrame chunk
for df_urb_pop in urb_pop_reader:
    # Check out specific country: df_pop_ceb
    df_pop_ceb = df_urb_pop[df_urb_pop['CountryCode'] == 'CEB']
    # Zip DataFrame columns of interest: pops
    pops = zip(df_pop_ceb['Total Population'],
                df_pop_ceb['Urban population (% of total)'])
    # Turn zip object into list: pops_list
    pops_list = list(pops)
    # Use list comprehension to create new DataFrame column 'Total Urban Population'
    df_pop_ceb['Total Urban Population'] = [int(tup[0] * tup[1] * 0.01) for tup in pops_list]
    # Append DataFrame chunk to data: data
    data = data.append(df_pop_ceb)
# Plot urban population data
data.plot(kind='scatter', x='Year', y='Total Urban Population')
plt.show()


### --------------------------------------------------------
# # ------>>>>> Writing an iterator to load data in chunks - ex#4
#--  --  --  -- Python Data Science Toolbox (Part 2): # Used for Data Scientist Training Path #FYI it's a compilation of how to work #with different commands. ### -------------------------------------------------------- # # ------>>>>>Iterators vs Iterables # Let's do a quick recall of what you've learned # about iterables and iterators. Recall from the video # that an iterable is an object that can return an iterator, #  while an iterator is an object that keeps state and produces # the next value when you call next() on it. In this exercise, # you will identify which object is an iterable and which is an iterator. # The environment has been pre-loaded with the variables flash1 and flash2. # Try printing out their values with print() and next() to figure out which # is an iterable and which is an iterator # R/ flash1 is an iterable and flash2 is an iterator. ### -------------------------------------------------------- # # ------>>>>> Iterating over iterables - ex#0 # Create a list of strings: flash # Print each list item in flash using a for loop # Create an iterator for flash: superspeed # Print each item from the iterator ### -------------------------------------------------------- # # ------>>>>> Iterating over iterables - ex#1 # Create an iterator for range(3): small_value # Print the values in small_value # Loop over range(3) and print the values # Create an iterator for range(10 ** 100): googol # Print the first 5 values from googol ### -------------------------------------------------------- # # ------>>>>> Iterators as function arguments # Create a range object: values # Print the range object # Create a list of integers: values_list # Print values_list # Get the sum of values: values_sum # Print values_sum ### -------------------------------------------------------- # Create a list of strings: mutants # Create a list of tuples: mutant_list # Print the list of tuples # Unpack and print the tuple pairs # Change the start index ### -------------------------------------------------------- # # ------>>>>> Using zip # Create a list of tuples: mutant_data # Print the list of tuples # Create a zip object using the three lists: mutant_zip # Print the zip object # Unpack the zip object and print the tuple values # Unpack the zip object and print the tuple values ### -------------------------------------------------------- # # ------>>>>> Using * and zip to 'unzip' # Create a zip object from mutants and powers: z1 # Print the tuples in z1 by unpacking with * # Re-create a zip object from mutants and powers: z1 # 'Unzip' the tuples in z1 by unpacking with * and zip(): result1, result2 # Check if unpacked tuples are equivalent to original tuples ### -------------------------------------------------------- # # ------>>>>> Processing large amounts of Twitter data # Initialize an empty dictionary: counts_dict # Iterate over the file chunk by chunk # Iterate over the column in DataFrame # Print the populated dictionary ### -------------------------------------------------------- # # ------>>>>> Extracting information for large amounts of Twitter data # Define count_entries() Return a dictionary with counts of
     occurrences as value for each key. # Initialize an empty dictionary: counts_dict # Iterate over the file chunk by chunk # Iterate over the column in DataFrame # Return counts_dict # Call count_entries(): result_counts # Print result_counts ### -------------------------------------------------------- # # ------>>>>> Write a basic list comprehension # In this exercise, you will practice what you've learned from # the video about writing list comprehensions. You will write a # list comprehension and identify the output that will be produced. # The following list has been pre-loaded in the environment. # doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson'] # How would a list comprehension that produces a list of the first # character of each string in doctor look like? Note that the list # comprehension uses doc as the iterator variable. What will the output be? # R/ The list comprehension is [doc[0] for doc in doctor] and produces # the list ['h', 'c', 'c', 't', 'w']. ### -------------------------------------------------------- # # ------>>>>> List comprehension over iterables # You know that list comprehensions can be built over iterables. # Given the following objects below, which of these can we build list comprehensions over? # doctor = ['house', 'cuddy', 'chase', 'thirteen', 'wilson'] # range(50) # underwood = 'After all, we are nothing more or less than what we choose to reveal.' # jean = '24601' # flash = ['<NAME>', '<NAME>', 'w<NAME>', '<NAME>'] # valjean = 24601 # R/ You can build list comprehensions over all the objects except the integer object valjean. ### -------------------------------------------------------- # # ------>>>>> Writing list comprehensions # Create list comprehension: squares ### -------------------------------------------------------- # # ------>>>>> Nested list comprehensions # Create a 5 x 5 matrix using a list of lists: matrix # Print the matrix ### -------------------------------------------------------- # # ------>>>>> Using conditionals in comprehensions - ex#0 # Create a list of strings: fellowship # Create list comprehension: new_fellowship # Print the new list ### -------------------------------------------------------- # # ------>>>>> Using conditionals in comprehensions - ex#1 # Create a list of strings: fellowship # Create list comprehension: new_fellowship # Print the new list ### -------------------------------------------------------- # # ------>>>>> Dict comprehensions # Create a list of strings: fellowship # Create dict comprehension: new_fellowship # Print the new list ### -------------------------------------------------------- # # ------>>>>> List comprehensions vs generators # You've seen from the videos that list comprehensions and generator # expressions look very similar in their syntax, except for the use of #  parentheses () in generator expressions and brackets [] in list comprehensions. # In this exercise, you will recall the difference between # list comprehensions and generators. To help with that task, # the following code has been pre-loaded in the environment: # # List of strings # fellowship = ['frodo', 'samwise', 'merry', 'aragorn', 'legolas', 'boromir', 'gimli'] # # List comprehension # fellow1 = [member for member in fellowship if len(member) >= 7] # # Generator expression # fellow2 = (member for member in fellowship if len(member) >= 7) # Try to play around with fellow1 and fellow2 by figuring out their types and # printing out their values. Based on your observations and what you can recall # from the video, select from the options below the best description for the # difference between list comprehensions and generators. # R/ A list comprehension produces a list as output, a generator produces a generator object. ### -------------------------------------------------------- # # ------>>>>> Write your own generator expressions # Create generator object: result # Print the first 5 values # Print the rest of the values ### -------------------------------------------------------- # # ------>>>>> Changing the output in generator expressions # Create a list of strings: lannister # Create a generator object: lengths # Iterate over and print the values in lengths ### -------------------------------------------------------- # # ------>>>>> Build a generator # Create a list of strings # Define generator function get_lengths Generator function that yields the
     length of the strings in input_list. # Yield the length of a string # Print the values generated by get_lengths() ### -------------------------------------------------------- # # ------>>>>> List comprehensions for time-stamped data # Extract the created_at column from df: tweet_time # Extract the clock time: tweet_clock_time # Print the extracted times ### -------------------------------------------------------- # # ------>>>>> Conditional list comprehensions for time-stamped data # Extract the created_at column from df: tweet_time # Extract the clock time: tweet_clock_time # Print the extracted times ### -------------------------------------------------------- # # ------>>>>> Dictionaries for data science # Zip lists: zipped_lists # Create a dictionary: rs_dict # Print the dictionary ### -------------------------------------------------------- # # ------>>>>> Writing a function to help you # Define lists2dict() Return a dictionary where list1 provides
     the keys and list2 provides the values. # Zip lists: zipped_lists # Create a dictionary: rs_dict # Return the dictionary # Call lists2dict: rs_fxn # Print rs_fxn ### -------------------------------------------------------- # # ------>>>>> Using a list comprehension # Print the first two lists in row_lists # Turn list of lists into list of dicts: list_of_dicts # Print the first two dictionaries in list_of_dicts ### -------------------------------------------------------- # # ------>>>>> Turning this all into a DataFrame # Import the pandas package # Turn list of lists into list of dicts: list_of_dicts # Turn list of dicts into a DataFrame: df # Print the head of the DataFrame ### -------------------------------------------------------- # # ------>>>>> Processing data in chunks - ex#0 # Skip the column names # Initialize an empty dictionary: counts_dict # Process only the first 1000 rows # Split the current line into a list: line # Get the value for the first column: first_col # If the column value is in the dict, increment its value # Else, add to the dict and set value to 1 # Print the resulting dictionary ### -------------------------------------------------------- # # ------>>>>> Writing a generator to load data in chunks (2) # Define read_large_file() A generator function to read a large file lazily. # Loop indefinitely until the end of the file # Read a line from the file: data # Break if this is the end of the file # Yield the line of data # Open a connection to the file # Create a generator object for the file: gen_file # Print the first three lines of the file ### -------------------------------------------------------- # # ------>>>>> Writing a generator to load data in chunks (3) # Initialize an empty dictionary: counts_dict # Open a connection to the file # Iterate over the generator from read_large_file() # Print ### -------------------------------------------------------- # # ------>>>>> Writing an iterator to load data in chunks - ex#0 # Import the pandas package # Initialize reader object: df_reader # Print two chunks ### -------------------------------------------------------- # # ------>>>>> Writing an iterator to load data in chunks - ex#1 # Initialize reader object: urb_pop_reader # Get the first DataFrame chunk: df_urb_pop # Check out the head of the DataFrame # Check out specific country: df_pop_ceb # Zip DataFrame columns of interest: pops # Turn zip object into list: pops_list # Print pops_list ### -------------------------------------------------------- # # ------>>>>> Writing an iterator to load data in chunks - ex#2 # Code from previous exercise # Use list comprehension to create new DataFrame column 'Total Urban Population' # Plot urban population data ### -------------------------------------------------------- # # ------>>>>> Writing an iterator to load data in chunks - ex#3 # Initialize reader object: urb_pop_reader # Initialize empty DataFrame: data # Iterate over each DataFrame chunk # Check out specific country: df_pop_ceb # Zip DataFrame columns of interest: pops # Turn zip object into list: pops_list # Use list comprehension to create new DataFrame column 'Total Urban Population' # Append DataFrame chunk to data: data # Plot urban population data ### -------------------------------------------------------- # # ------>>>>> Writing an iterator to load data in chunks - ex#4