original_docstring: 
          Runs an episode of simulation, then starts a training session
          
docstring: Runs an episode of simulation, then starts a training session
docstring_tokens: Runs an episode of simulation then starts a training session
code: def run(self, episode, epsilon):
          """
          Runs an episode of simulation, then starts a training session
          """
          start_time = timeit.default_timer()
  
          # first, generate the route file for this simulation and set up sumo
          self._TrafficGen.generate_routefile(seed=episode)
          traci.start(self._sumo_cmd)
          print("Simulating...")
  
          # inits
          self._step = 0
          self._waiting_times = {}
          self._sum_neg_reward = 0
          self._sum_queue_length = 0
          self._sum_waiting_time = 0
          self._model_training_loss = []
          self._already_in = []
          self._density = []
          self._flow = []
          self._occupancy = []
          self._cumulative_waiting_time = 0
          old_total_wait = 0
          old_state = -1
          old_action = -1
  
          # run the simulation
          while self._step < self._max_steps:
  
              # get current state of the intersection
              current_state = self._get_state_with_advanced_perception()
              
              # calculate reward of previous action: (change in cumulative waiting time between actions)
              # waiting time = seconds waited by a car since the spawn in the environment, cumulated for every car in incoming lanes
              current_total_wait = self._collect_waiting_times()
              reward = old_total_wait - current_total_wait
              #reward = (0.9 * old_total_wait) - current_total_wait
              
              self._cumulative_waiting_time+= current_total_wait #cumulative waiting time
  
              # saving the data into the memory
              if self._step != 0:
                  #self._Memory.add_sample((old_state, old_action, reward, current_state))
                  #print(type(old_state), type(old_action), type(reward), type(current_state))
                  #print(type(old_state.tolist()))
                  requests.post('http://127.0.0.1:5000/add_sample', json={'old_state':  old_state.tolist(),
                                                                      'old_action': int(old_action),
                                                                      'reward': reward,
                                                                      'current_state': current_state.tolist()})
  
              # choose the light phase to activate, based on the current state of the intersection
              action = self._choose_action(current_state, epsilon)
  
              # if the chosen phase is different from the last phase, activate the yellow phase
              if self._step != 0 and old_action != action:
                  self._set_yellow_phase(old_action)
                  self._simulate(self._yellow_duration)
  
              # execute the phase selected before
              self._set_green_phase(action)
              self._simulate(self._green_duration)
  
              # saving variables for later & accumulate reward
              old_state = current_state
              old_action = action
              old_total_wait = current_total_wait
  
              # saving only the meaningful reward to better see if the agent is behaving correctly
              if reward < 0:
                  self._sum_neg_reward += reward
  
          print("Saving episodes stats...")
          self._save_episode_stats()
          print("Total reward:", self._sum_neg_reward, "- Epsilon:", round(epsilon, 2))
          traci.close()
          simulation_time = round(timeit.default_timer() - start_time, 1)
  
          # print("Training...")
          # start_time = timeit.default_timer()
          # for _ in range(self._training_epochs):
          #     #self._replay()
          #     tr_loss = requests.post('http://127.0.0.1:5000/replay', json={'num_states': self._num_states,
          #                                                       'num_actions': self._num_actions,
          #                                                       'gamma': self._gamma}).json()['loss']
          #     #print(tr_loss)
          #     self._model_training_loss.append(tr_loss)
          # training_time = round(timeit.default_timer() - start_time, 1)
          
          # if(len(self._model_training_loss) > 0):
          #     print("Saving loss results...")
          #     #print(self._model_training_loss)
          #     self._avg_loss.append(sum(self._model_training_loss)/self._training_epochs)
          #     self._min_loss.append(min(self._model_training_loss))
  
          return simulation_time
code_tokens: def run ( self , episode , epsilon ) : start_time = timeit . default_timer ( ) self . _TrafficGen . generate_routefile ( seed = episode ) traci . start ( self . _sumo_cmd ) print ( "Simulating..." ) self . _step = 0 self . _waiting_times = { } self . _sum_neg_reward = 0 self . _sum_queue_length = 0 self . _sum_waiting_time = 0 self . _model_training_loss = [ ] self . _already_in = [ ] self . _density = [ ] self . _flow = [ ] self . _occupancy = [ ] self . _cumulative_waiting_time = 0 old_total_wait = 0 old_state = - 1 old_action = - 1 while self . _step < self . _max_steps : current_state = self . _get_state_with_advanced_perception ( ) current_total_wait = self . _collect_waiting_times ( ) reward = old_total_wait - current_total_wait self . _cumulative_waiting_time += current_total_wait if self . _step != 0 : requests . post ( 'http://127.0.0.1:5000/add_sample' , json = { 'old_state' : old_state . tolist ( ) , 'old_action' : int ( old_action ) , 'reward' : reward , 'current_state' : current_state . tolist ( ) } ) action = self . _choose_action ( current_state , epsilon ) if self . _step != 0 and old_action != action : self . _set_yellow_phase ( old_action ) self . _simulate ( self . _yellow_duration ) self . _set_green_phase ( action ) self . _simulate ( self . _green_duration ) old_state = current_state old_action = action old_total_wait = current_total_wait if reward < 0 : self . _sum_neg_reward += reward print ( "Saving episodes stats..." ) self . _save_episode_stats ( ) print ( "Total reward:" , self . _sum_neg_reward , "- Epsilon:" , round ( epsilon , 2 ) ) traci . close ( ) simulation_time = round ( timeit . default_timer ( ) - start_time , 1 ) return simulation_time
short_docstring: Runs an episode of simulation, then starts a training session
short_docstring_tokens: Runs an episode of simulation then starts a training session
comment: """
          Runs an episode of simulation, then starts a training session
          """$SEP$# first, generate the route file for this simulation and set up sumo$SEP$# inits$SEP$# run the simulation$SEP$# get current state of the intersection$SEP$# calculate reward of previous action: (change in cumulative waiting time between actions)$SEP$# waiting time = seconds waited by a car since the spawn in the environment, cumulated for every car in incoming lanes$SEP$#reward = (0.9 * old_total_wait) - current_total_wait$SEP$#cumulative waiting time$SEP$# saving the data into the memory$SEP$#self._Memory.add_sample((old_state, old_action, reward, current_state))$SEP$#print(type(old_state), type(old_action), type(reward), type(current_state))$SEP$#print(type(old_state.tolist()))$SEP$# choose the light phase to activate, based on the current state of the intersection$SEP$# if the chosen phase is different from the last phase, activate the yellow phase$SEP$# execute the phase selected before$SEP$# saving variables for later & accumulate reward$SEP$# saving only the meaningful reward to better see if the agent is behaving correctly$SEP$# print("Training...")$SEP$# start_time = timeit.default_timer()$SEP$# for _ in range(self._training_epochs):$SEP$#     #self._replay()$SEP$#     tr_loss = requests.post('http://127.0.0.1:5000/replay', json={'num_states': self._num_states,$SEP$#                                                       'num_actions': self._num_actions,$SEP$#                                                       'gamma': self._gamma}).json()['loss']$SEP$#     #print(tr_loss)$SEP$#     self._model_training_loss.append(tr_loss)$SEP$# training_time = round(timeit.default_timer() - start_time, 1)$SEP$# if(len(self._model_training_loss) > 0):$SEP$#     print("Saving loss results...")$SEP$#     #print(self._model_training_loss)$SEP$#     self._avg_loss.append(sum(self._model_training_loss)/self._training_epochs)$SEP$#     self._min_loss.append(min(self._model_training_loss))