<reponame>BridgeQZH/Deep-QLearning-Agent-for-Traffic-Signal-Control
import traci
import numpy as np
import random
import timeit
import os
from reward_by_hand import g_function

# phase codes based on environment.net.xml
PHASE_NS_GREEN = 0  # action 0 code 00
PHASE_NS_YELLOW = 1
PHASE_NSL_GREEN = 2  # action 1 code 01
PHASE_NSL_YELLOW = 3
PHASE_EW_GREEN = 4  # action 2 code 10
PHASE_EW_YELLOW = 5
PHASE_EWL_GREEN = 6  # action 3 code 11
PHASE_EWL_YELLOW = 7


class Simulation:
    def __init__(self, Model, TrafficGen, sumo_cmd, num_vehicles, max_steps, green_duration, yellow_duration, num_states, num_actions):
        self._Model = Model
        self._TrafficGen = TrafficGen
        self._step = 0
        self._sumo_cmd = sumo_cmd
        self._num_vehicles = num_vehicles
        self._max_steps = max_steps
        self._green_duration = green_duration
        self._yellow_duration = yellow_duration
        self._num_states = num_states
        self._num_actions = num_actions
        self._reward_episode = []
        self._queue_length_episode = []
        


    def run(self, episode):
        """
        Runs the testing simulation
        """
        start_time = timeit.default_timer()

        # first, generate the route file for this simulation and set up sumo
        self._TrafficGen.generate_routefile(seed=episode)
        traci.start(self._sumo_cmd)
        print("Simulating...")

        # inits
        self._step = 0
        self._waiting_times = {}
        old_total_wait = 0
        old_action = -1 # dummy init

        while self._step < self._max_steps:

            # get current state of the intersection
            current_state = self._get_state()

            # calculate reward of previous action: (change in cumulative waiting time between actions)
            # waiting time = seconds waited by a car since the spawn in the environment, cumulated for every car in incoming lanes
            current_total_wait = self._collect_waiting_times()
            # reward = (old_total_wait - current_total_wait) # TODO: consider the comparison

            # choose the light phase to activate, based on the current state of the intersection
            action = self._choose_action(current_state)

            reward = g_function(current_state, action, old_action)
            print("step:",  self._step, "reward:", reward)
            if action == 0:
                print("action is North and South Green")
            if action == 1:
                print("action is North and South Left Green")
            if action == 2:
                print("action is East and West Green")
            if action == 3:
                print("action is East and West Left Green")
            # if the chosen phase is different from the last phase, activate the yellow phase
            if self._step != 0 and old_action != action:
                self._set_yellow_phase(old_action)
                self._simulate(self._yellow_duration)

            # execute the phase selected before
            self._set_green_phase(action)
            self._simulate(self._green_duration)

            # saving variables for later & accumulate reward
            old_action = action
            old_total_wait = current_total_wait

            self._reward_episode.append(reward)
            print("step:", self._step, "current step reward:", reward)

        # print("Total reward:", np.sum(self._reward_episode)) # A very small number
        traci.close()
        simulation_time = round(timeit.default_timer() - start_time, 1)

        return simulation_time


    def _simulate(self, steps_todo):
        """
        Proceed with the simulation in sumo
        """
        if (self._step + steps_todo) >= self._max_steps:  # do not do more steps than the maximum allowed number of steps
            steps_todo = self._max_steps - self._step

        while steps_todo > 0:
            traci.simulationStep()  # simulate 1 step in sumo
            self._step += 1 # update the step counter
            steps_todo -= 1
            queue_length = self._get_queue_length()
            self._queue_length_episode.append(queue_length)


    def _collect_waiting_times(self):
        """
        Retrieve the waiting time of every car in the incoming roads
        """
        incoming_roads = ["E2TL", "N2TL", "W2TL", "S2TL"]
        car_list = traci.vehicle.getIDList()
        for car_id in car_list:
            wait_time = traci.vehicle.getAccumulatedWaitingTime(car_id)
            road_id = traci.vehicle.getRoadID(car_id)  # get the road id where the car is located
            if road_id in incoming_roads:  # consider only the waiting times of cars in incoming roads
                self._waiting_times[car_id] = wait_time
            else:
                if car_id in self._waiting_times: # a car that was tracked has cleared the intersection
                    del self._waiting_times[car_id] 
        total_waiting_time = sum(self._waiting_times.values())
        return total_waiting_time


    def _choose_action(self, state):
        """
        Pick the best action known based on the current state of the env
        model.predict returns numpy array(s) of predictions
        """
        print("Max among four Q", np.max(self._Model.predict_one(state)))
        print("Full four Q", self._Model.predict_one(state))
        return np.argmax(self._Model.predict_one(state))


    def _set_yellow_phase(self, old_action):
        """
        Activate the correct yellow light combination in sumo
        """
        yellow_phase_code = old_action * 2 + 1 # obtain the yellow phase code, based on the old action (ref on environment.net.xml)
        traci.trafficlight.setPhase("TL", yellow_phase_code)


    def _set_green_phase(self, action_number):
        """
        Activate the correct green light combination in sumo
        """
        if action_number == 0:
            traci.trafficlight.setPhase("TL", PHASE_NS_GREEN)
        elif action_number == 1:
            traci.trafficlight.setPhase("TL", PHASE_NSL_GREEN)
        elif action_number == 2:
            traci.trafficlight.setPhase("TL", PHASE_EW_GREEN)
        elif action_number == 3:
            traci.trafficlight.setPhase("TL", PHASE_EWL_GREEN)


    def _get_queue_length(self):
        """
        Retrieve the number of cars with speed = 0 in every incoming lane
        """
        halt_N = traci.edge.getLastStepHaltingNumber("N2TL")
        halt_S = traci.edge.getLastStepHaltingNumber("S2TL")
        halt_E = traci.edge.getLastStepHaltingNumber("E2TL")
        halt_W = traci.edge.getLastStepHaltingNumber("W2TL")
        queue_length = halt_N + halt_S + halt_E + halt_W
        return queue_length


    def _get_state(self):
        """
        Retrieve the state of the intersection from sumo, in the form of cell occupancy
        """
        state = np.zeros(self._num_states) # Occupy matrix - 80; Number of vehicles - 16 or 12
        car_list = traci.vehicle.getIDList()

        for car_id in car_list:
            lane_pos = traci.vehicle.getLanePosition(car_id)
            lane_pos = 750 - lane_pos  # inversion of lane pos, so if the car is close to the traffic light -> lane_pos = 0 --- 750 = max len of a road
            lane_id = traci.vehicle.getLaneID(car_id) # Returns the id of the lane the named vehicle was at within the last step.
            if lane_pos <= 100:
                if lane_id == "N2TL_0":
                    state[0] += 1
                    
                elif lane_id == "N2TL_1" or lane_id == "N2TL_2":
                    state[1] += 1
                    
                elif lane_id == "N2TL_3":
                    state[2] += 1
                    
                elif lane_id == "S2TL_0":
                    state[3] += 1
                    
                elif lane_id == "S2TL_1" or lane_id == "S2TL_2":
                    state[4] += 1
                    
                elif lane_id == "S2TL_3":
                    state[5] += 1
                    
                elif lane_id == "E2TL_0":
                    state[6] += 1
                    
                elif lane_id == "E2TL_1" or lane_id == "E2TL_2":
                    state[7] += 1
                    
                elif lane_id == "E2TL_3":
                    state[8] += 1
                    
                elif lane_id == "W2TL_0":
                    state[9] += 1
                    
                elif lane_id == "W2TL_1" or lane_id == "W2TL_2":
                    state[10] += 1
                    
                elif lane_id == "W2TL_3":
                    state[11] += 1

        return state


    @property
    def queue_length_episode(self):
        return self._queue_length_episode


    @property
    def reward_episode(self):
        accu_reward = []
        for i in range(len(self._reward_episode)):
            b1 = sum(self._reward_episode[0:i+1])
            accu_reward.append(b1)
        return accu_reward




import traci
import numpy as np
import random
import timeit
import os
from reward_by_hand import g_function

# phase codes based on environment.net.xml
PHASE_NS_GREEN = 0  # action 0 code 00
PHASE_NS_YELLOW = 1
PHASE_NSL_GREEN = 2  # action 1 code 01
PHASE_NSL_YELLOW = 3
PHASE_EW_GREEN = 4  # action 2 code 10
PHASE_EW_YELLOW = 5
PHASE_EWL_GREEN = 6  # action 3 code 11
PHASE_EWL_YELLOW = 7


class Simulation:
    def __init__(self, Model, TrafficGen, sumo_cmd, num_vehicles, max_steps, green_duration, yellow_duration, num_states, num_actions):
        self._Model = Model
        self._TrafficGen = TrafficGen
        self._step = 0
        self._sumo_cmd = sumo_cmd
        self._num_vehicles = num_vehicles
        self._max_steps = max_steps
        self._green_duration = green_duration
        self._yellow_duration = yellow_duration
        self._num_states = num_states
        self._num_actions = num_actions
        self._reward_episode = []
        self._queue_length_episode = []
        


    def run(self, episode):
        """
        Runs the testing simulation
        """
        start_time = timeit.default_timer()

        # first, generate the route file for this simulation and set up sumo
        self._TrafficGen.generate_routefile(seed=episode)
        traci.start(self._sumo_cmd)
        print("Simulating...")

        # inits
        self._step = 0
        self._waiting_times = {}
        old_total_wait = 0
        old_action = -1 # dummy init

        while self._step < self._max_steps:

            # get current state of the intersection
            current_state = self._get_state()

            # calculate reward of previous action: (change in cumulative waiting time between actions)
            # waiting time = seconds waited by a car since the spawn in the environment, cumulated for every car in incoming lanes
            current_total_wait = self._collect_waiting_times()
            # reward = (old_total_wait - current_total_wait) # TODO: consider the comparison

            # choose the light phase to activate, based on the current state of the intersection
            action = self._choose_action(current_state)

            reward = g_function(current_state, action, old_action)
            print("step:",  self._step, "reward:", reward)
            if action == 0:
                print("action is North and South Green")
            if action == 1:
                print("action is North and South Left Green")
            if action == 2:
                print("action is East and West Green")
            if action == 3:
                print("action is East and West Left Green")
            # if the chosen phase is different from the last phase, activate the yellow phase
            if self._step != 0 and old_action != action:
                self._set_yellow_phase(old_action)
                self._simulate(self._yellow_duration)

            # execute the phase selected before
            self._set_green_phase(action)
            self._simulate(self._green_duration)

            # saving variables for later & accumulate reward
            old_action = action
            old_total_wait = current_total_wait

            self._reward_episode.append(reward)
            print("step:", self._step, "current step reward:", reward)

        # print("Total reward:", np.sum(self._reward_episode)) # A very small number
        traci.close()
        simulation_time = round(timeit.default_timer() - start_time, 1)

        return simulation_time


    def _simulate(self, steps_todo):
        """
        Proceed with the simulation in sumo
        """
        if (self._step + steps_todo) >= self._max_steps:  # do not do more steps than the maximum allowed number of steps
            steps_todo = self._max_steps - self._step

        while steps_todo > 0:
            traci.simulationStep()  # simulate 1 step in sumo
            self._step += 1 # update the step counter
            steps_todo -= 1
            queue_length = self._get_queue_length()
            self._queue_length_episode.append(queue_length)


    def _collect_waiting_times(self):
        """
        Retrieve the waiting time of every car in the incoming roads
        """
        incoming_roads = ["E2TL", "N2TL", "W2TL", "S2TL"]
        car_list = traci.vehicle.getIDList()
        for car_id in car_list:
            wait_time = traci.vehicle.getAccumulatedWaitingTime(car_id)
            road_id = traci.vehicle.getRoadID(car_id)  # get the road id where the car is located
            if road_id in incoming_roads:  # consider only the waiting times of cars in incoming roads
                self._waiting_times[car_id] = wait_time
            else:
                if car_id in self._waiting_times: # a car that was tracked has cleared the intersection
                    del self._waiting_times[car_id] 
        total_waiting_time = sum(self._waiting_times.values())
        return total_waiting_time


    def _choose_action(self, state):
        """
        Pick the best action known based on the current state of the env
        model.predict returns numpy array(s) of predictions
        """
        print("Max among four Q", np.max(self._Model.predict_one(state)))
        print("Full four Q", self._Model.predict_one(state))
        return np.argmax(self._Model.predict_one(state))


    def _set_yellow_phase(self, old_action):
        """
        Activate the correct yellow light combination in sumo
        """
        yellow_phase_code = old_action * 2 + 1 # obtain the yellow phase code, based on the old action (ref on environment.net.xml)
        traci.trafficlight.setPhase("TL", yellow_phase_code)


    def _set_green_phase(self, action_number):
        """
        Activate the correct green light combination in sumo
        """
        if action_number == 0:
            traci.trafficlight.setPhase("TL", PHASE_NS_GREEN)
        elif action_number == 1:
            traci.trafficlight.setPhase("TL", PHASE_NSL_GREEN)
        elif action_number == 2:
            traci.trafficlight.setPhase("TL", PHASE_EW_GREEN)
        elif action_number == 3:
            traci.trafficlight.setPhase("TL", PHASE_EWL_GREEN)


    def _get_queue_length(self):
        """
        Retrieve the number of cars with speed = 0 in every incoming lane
        """
        halt_N = traci.edge.getLastStepHaltingNumber("N2TL")
        halt_S = traci.edge.getLastStepHaltingNumber("S2TL")
        halt_E = traci.edge.getLastStepHaltingNumber("E2TL")
        halt_W = traci.edge.getLastStepHaltingNumber("W2TL")
        queue_length = halt_N + halt_S + halt_E + halt_W
        return queue_length


    def _get_state(self):
        """
        Retrieve the state of the intersection from sumo, in the form of cell occupancy
        """
        state = np.zeros(self._num_states) # Occupy matrix - 80; Number of vehicles - 16 or 12
        car_list = traci.vehicle.getIDList()

        for car_id in car_list:
            lane_pos = traci.vehicle.getLanePosition(car_id)
            lane_pos = 750 - lane_pos  # inversion of lane pos, so if the car is close to the traffic light -> lane_pos = 0 --- 750 = max len of a road
            lane_id = traci.vehicle.getLaneID(car_id) # Returns the id of the lane the named vehicle was at within the last step.
            if lane_pos <= 100:
                if lane_id == "N2TL_0":
                    state[0] += 1
                    
                elif lane_id == "N2TL_1" or lane_id == "N2TL_2":
                    state[1] += 1
                    
                elif lane_id == "N2TL_3":
                    state[2] += 1
                    
                elif lane_id == "S2TL_0":
                    state[3] += 1
                    
                elif lane_id == "S2TL_1" or lane_id == "S2TL_2":
                    state[4] += 1
                    
                elif lane_id == "S2TL_3":
                    state[5] += 1
                    
                elif lane_id == "E2TL_0":
                    state[6] += 1
                    
                elif lane_id == "E2TL_1" or lane_id == "E2TL_2":
                    state[7] += 1
                    
                elif lane_id == "E2TL_3":
                    state[8] += 1
                    
                elif lane_id == "W2TL_0":
                    state[9] += 1
                    
                elif lane_id == "W2TL_1" or lane_id == "W2TL_2":
                    state[10] += 1
                    
                elif lane_id == "W2TL_3":
                    state[11] += 1

        return state


    @property
    def queue_length_episode(self):
        return self._queue_length_episode


    @property
    def reward_episode(self):
        accu_reward = []
        for i in range(len(self._reward_episode)):
            b1 = sum(self._reward_episode[0:i+1])
            accu_reward.append(b1)
        return accu_reward
# phase codes based on environment.net.xml # action 0 code 00 # action 1 code 01 # action 2 code 10 # action 3 code 11 Runs the testing simulation # first, generate the route file for this simulation and set up sumo # inits # dummy init # get current state of the intersection # calculate reward of previous action: (change in cumulative waiting time between actions) # waiting time = seconds waited by a car since the spawn in the environment, cumulated for every car in incoming lanes # reward = (old_total_wait - current_total_wait) # TODO: consider the comparison # choose the light phase to activate, based on the current state of the intersection # if the chosen phase is different from the last phase, activate the yellow phase # execute the phase selected before # saving variables for later & accumulate reward # print("Total reward:", np.sum(self._reward_episode)) # A very small number Proceed with the simulation in sumo # do not do more steps than the maximum allowed number of steps # simulate 1 step in sumo # update the step counter Retrieve the waiting time of every car in the incoming roads # get the road id where the car is located # consider only the waiting times of cars in incoming roads # a car that was tracked has cleared the intersection Pick the best action known based on the current state of the env         model.predict returns numpy array(s) of predictions Activate the correct yellow light combination in sumo # obtain the yellow phase code, based on the old action (ref on environment.net.xml) Activate the correct green light combination in sumo Retrieve the number of cars with speed = 0 in every incoming lane Retrieve the state of the intersection from sumo, in the form of cell occupancy # Occupy matrix - 80; Number of vehicles - 16 or 12 # inversion of lane pos, so if the car is close to the traffic light -> lane_pos = 0 --- 750 = max len of a road # Returns the id of the lane the named vehicle was at within the last step.