lib/tasks/crawler_fin_calendar.rake
# Runs through a CSV list of allegedly updated meetings, obtained from running
# the dedicated JS crawler 'fin-crawler.js', and returns
#
# === CSV format for parsing:
#
# startURL,date,isCancelled,name,place,meetingUrl,year
#
# === Returns
#
# An Hash of FinCalendar rows, one for each element of the result.
#
Runs through a CSV list of allegedly updated meetings, obtained from running
the dedicated JS crawler 'fin-crawler.js', and returns

CSV format for parsing.

startURL,date,isCancelled,name,place,meetingUrl,year

Returns

An Hash of FinCalendar rows, one for each element of the result.
Runs through a CSV list of allegedly updated meetings obtained from running the dedicated JS crawler ' fin - crawler . js ' and returns CSV format for parsing . startURL date isCancelled name place meetingUrl year Returns An Hash of FinCalendar rows one for each element of the result .
def from_csv_meeting_list_to_calendar_rows( meeting_list_on_csv_text )
    output_path = create_base_output_path_if_missing()
    api_results_endpoint = start_apifier_crawler( api_run_endpoint )
    json_result = get_results_from_apifier_crawler( api_results_endpoint )
    calendar_rows = {}
    result_list = json_result.each do |parsed_hash|
      label = parsed_hash['label']
      current_url = parsed_hash['url']
      puts "\r\n\r\nProcessing results for season #{ label }"
      puts "(#{ current_url }) => #{ parsed_hash['pageFunctionResult'].size } tot. links"
      puts " "
      full_output_path = File.join(output_path, label)
      FileUtils.mkdir_p( full_output_path ) if !File.directory?( full_output_path )
      parsed_hash['pageFunctionResult'].each_with_index do |row_hash, index|
        year  = row_hash['year']  
        month = row_hash['month']
        days  = row_hash['days']
        city  = row_hash['city']
        description = row_hash['description']
        link = "https://www.federnuoto.it#{ row_hash['link'] }"
        coded_date_key = FinCalendar.calendar_unique_key( year, month, days, city )
        puts "Processing #{ index + 1 }/#{ parsed_hash['pageFunctionResult'].size } => #{ coded_date_key }..."
        calendar_row = if calendar_rows.has_key?( coded_date_key )
          calendar_rows[ coded_date_key ]
        else
          FinCalendar.new(
            season_id: label.to_i,
            calendar_year: year,
            calendar_month: month,
            calendar_date: days,
            calendar_place: city
          )
        end
        calendar_row.season_id      ||= label.to_i
        calendar_row.calendar_year  ||= year
        calendar_row.calendar_month ||= month
        calendar_row.calendar_date  ||= days
        calendar_row.calendar_place ||= city
        calendar_row.goggles_meeting_code ||= NameNormalizer.get_meeting_code( description, city )
        if description =~ /risultati/i              
          calendar_row.results_link   = row_hash['link']
          uri = URI( row_hash['link'] )
          params_array = URI.decode_www_form( uri.query )
          fin_code = params_array.select{ |e| e.member?('codice') }.flatten.last
          calendar_row.fin_results_code ||= fin_code
        elsif description =~ /start/i               
          calendar_row.startlist_link = row_hash['link']
          uri = URI( row_hash['link'] )
          params_array = URI.decode_www_form( uri.query )
          fin_code = params_array.select{ |e| e.member?('codice') }.flatten.last
          calendar_row.fin_startlist_code ||= fin_code
        else                                        
          calendar_row.manifest_link  = row_hash['link']
          calendar_row.calendar_name  = description
          calendar_row.goggles_meeting_code ||= NameNormalizer.get_meeting_code( description, city )
        end
        calendar_rows[ coded_date_key ] = calendar_row
      end
    end
    puts "Scanning done."
    calendar_rows
  end
def from_csv_meeting_list_to_calendar_rows ( meeting_list_on_csv_text ) output_path = create_base_output_path_if_missing ( ) api_results_endpoint = start_apifier_crawler ( api_run_endpoint ) json_result = get_results_from_apifier_crawler ( api_results_endpoint ) calendar_rows = { } result_list = json_result . each do | parsed_hash | label = parsed_hash [ 'label' ] current_url = parsed_hash [ 'url' ] puts "\r\n\r\nProcessing results for season #{ label }" puts "(#{ current_url }) => #{ parsed_hash['pageFunctionResult'].size } tot. links" puts " " full_output_path = File . join ( output_path , label ) FileUtils . mkdir_p ( full_output_path ) if ! File . directory? ( full_output_path ) parsed_hash [ 'pageFunctionResult' ] . each_with_index do | row_hash , index | year = row_hash [ 'year' ] month = row_hash [ 'month' ] days = row_hash [ 'days' ] city = row_hash [ 'city' ] description = row_hash [ 'description' ] link = "https://www.federnuoto.it#{ row_hash['link'] }" coded_date_key = FinCalendar . calendar_unique_key ( year , month , days , city ) puts "Processing #{ index + 1 }/#{ parsed_hash['pageFunctionResult'].size } => #{ coded_date_key }..." calendar_row = if calendar_rows . has_key? ( coded_date_key ) calendar_rows [ coded_date_key ] else FinCalendar . new ( season_id : label . to_i , calendar_year : year , calendar_month : month , calendar_date : days , calendar_place : city ) end calendar_row . season_id ||= label . to_i calendar_row . calendar_year ||= year calendar_row . calendar_month ||= month calendar_row . calendar_date ||= days calendar_row . calendar_place ||= city calendar_row . goggles_meeting_code ||= NameNormalizer . get_meeting_code ( description , city ) if description =~ / risultati /i calendar_row . results_link = row_hash [ 'link' ] uri = URI ( row_hash [ 'link' ] ) params_array = URI . decode_www_form ( uri . query ) fin_code = params_array . select { | e | e . member? ( 'codice' ) } . flatten . last calendar_row . fin_results_code ||= fin_code elsif description =~ / start /i calendar_row . startlist_link = row_hash [ 'link' ] uri = URI ( row_hash [ 'link' ] ) params_array = URI . decode_www_form ( uri . query ) fin_code = params_array . select { | e | e . member? ( 'codice' ) } . flatten . last calendar_row . fin_startlist_code ||= fin_code else calendar_row . manifest_link = row_hash [ 'link' ] calendar_row . calendar_name = description calendar_row . goggles_meeting_code ||= NameNormalizer . get_meeting_code ( description , city ) end calendar_rows [ coded_date_key ] = calendar_row end end puts "Scanning done." calendar_rows end
Runs through a CSV list of allegedly updated meetings, obtained from running
the dedicated JS crawler 'fin-crawler.js', and returns
Runs through a CSV list of allegedly updated meetings obtained from running the dedicated JS crawler ' fin - crawler . js ' and returns
# Start the crawler and wait for the results to be ready: # Retrieve the results: # Build up the result list: # Hash of *unique* calendar rows: key = coded_suffix, value = calendar row # DEBUG #    puts "json_result => #{ json_result.class }" #    puts "json_result.first => #{ json_result.first.class }\r\n" # ASSERT: pageFunctionResult is an array of Hash: # Create the output sub-path, if missing: # For each calendar URL, we have several row result Hash instances: # Year MUST always come from the calendar, not from parsing the Link # Compose the full URL for the extracted sub,links: # Use a coded key for the Hash list: # Overwrite nil fields with new values in case they are missing: # - MEETING RESULTS - # - EXTRACT FIN CODE from link: # - MEETING START-LIST - # - EXTRACT FIN CODE from link: # - MEETING MANIFEST - # (Re-)Set the updated row into the parsed hash:
meeting_list_on_csv_text