original_docstring: Creates a list of url payload tuples
docstring: Creates a list of url payload tuples
docstring_tokens: Creates a list of url payload tuples
code: def gen_url_payloads(self, url, batch_size, search_window):
          """Creates a list of url payload tuples"""
          url_payloads = []
  
          # check if new payloads have to be made
          if len(self.req_list) == 0:
              # paging for ids
              if 'ids' in self.payload:
  
                  # convert ids to list
                  self._id_list(self.payload)
  
                  all_ids = self.payload['ids']
                  if len(all_ids) == 0 and (self.limit and self.limit > 0):
                      warnings.warn(
                          f'{self.limit} items were not found in Pushshift')
                  self.limit = len(all_ids)
  
                  # remove ids from payload to prevent , -> %2C and increasing query length
                  # beyond the max length of 8190
                  self.payload['ids'] = []
  
                  # if searching for submission comment ids
                  if self.kind == "submission_comment_ids":
                      urls = [url+sub_id for sub_id in all_ids]
                      url_payloads = [(url, self.payload) for url in urls]
                  else:
                      # split ids into arrays of size max_ids_per_request
                      ids_split = []
                      max_len = self.max_ids_per_request
                      while len(all_ids) > 0:
                          ids_split.append(",".join(all_ids[:max_len]))
                          all_ids = all_ids[max_len:]
  
                      log.debug(f'Created {len(ids_split)} id slices')
  
                      # create url payload tuples
                      url_payloads = [(url + '?ids=' + id_str, self.payload)
                                      for id_str in ids_split]
                  # add payloads to req_list
                  self.req_list.extend(url_payloads)
  
              else:
                  if 'after' not in self.payload:
                      search_window = dt.timedelta(days=search_window)
                      num = batch_size
                      before = self.payload['before']
                      after = int((dt.datetime.fromtimestamp(
                          before) - search_window).timestamp())
  
                      # set before to after for future time slices
                      self.payload['before'] = after
  
                  else:
                      before = self.payload['before']
                      after = self.payload['after']
  
                      # set before to avoid repeated time slices when there are missed responses
                      self.payload['before'] = after
                      num = batch_size
  
                  # generate payloads
                  self.gen_slices(
                      url, self.payload, after, before, num)
code_tokens: def gen_url_payloads ( self , url , batch_size , search_window ) : url_payloads = [ ] if len ( self . req_list ) == 0 : if 'ids' in self . payload : self . _id_list ( self . payload ) all_ids = self . payload [ 'ids' ] if len ( all_ids ) == 0 and ( self . limit and self . limit > 0 ) : warnings . warn ( f'{self.limit} items were not found in Pushshift' ) self . limit = len ( all_ids ) self . payload [ 'ids' ] = [ ] if self . kind == "submission_comment_ids" : urls = [ url + sub_id for sub_id in all_ids ] url_payloads = [ ( url , self . payload ) for url in urls ] else : ids_split = [ ] max_len = self . max_ids_per_request while len ( all_ids ) > 0 : ids_split . append ( "," . join ( all_ids [ : max_len ] ) ) all_ids = all_ids [ max_len : ] log . debug ( f'Created {len(ids_split)} id slices' ) url_payloads = [ ( url + '?ids=' + id_str , self . payload ) for id_str in ids_split ] self . req_list . extend ( url_payloads ) else : if 'after' not in self . payload : search_window = dt . timedelta ( days = search_window ) num = batch_size before = self . payload [ 'before' ] after = int ( ( dt . datetime . fromtimestamp ( before ) - search_window ) . timestamp ( ) ) self . payload [ 'before' ] = after else : before = self . payload [ 'before' ] after = self . payload [ 'after' ] self . payload [ 'before' ] = after num = batch_size self . gen_slices ( url , self . payload , after , before , num )
short_docstring: Creates a list of url payload tuples
short_docstring_tokens: Creates a list of url payload tuples
comment: """Creates a list of url payload tuples"""$SEP$# check if new payloads have to be made$SEP$# paging for ids$SEP$# convert ids to list$SEP$# remove ids from payload to prevent , -> %2C and increasing query length$SEP$# beyond the max length of 8190$SEP$# if searching for submission comment ids$SEP$# split ids into arrays of size max_ids_per_request$SEP$# create url payload tuples$SEP$# add payloads to req_list$SEP$# set before to after for future time slices$SEP$# set before to avoid repeated time slices when there are missed responses$SEP$# generate payloads