app/models/crawler/data_feeder.rb
# NOTE: inefficient to open and close the file for every taxon... but
# this allows us to see partial results sooner, and we don't mind the
# additional pause (because our queries slow down the site):
inefficient to open and close the file for every taxon... but
this allows us to see partial results sooner, and we don't mind the
additional pause (because our queries slow down the site).
inefficient to open and close the file for every taxon ... but this allows us to see partial results sooner and we don ' t mind the additional pause ( because our queries slow down the site ) .
def add_json(name, json)
      file = File.open(name, "r+")
      begin
        pos_stack = [0, 0, 0]
        content_stack = ["", ""]
        file.each do |line|
          unless file.eof?
            pos_stack.push(file.pos)
            pos_stack.shift
            content_stack.push(line)
            content_stack.shift
          end
        end
        file.seek(pos_stack.first, IO::SEEK_SET)
        prev_content = content_stack.first.chomp!
        prev_content += "," unless prev_content[-1] == "["
        file.puts("#{prev_content}")
        formatted = JSON.pretty_generate(json).gsub(/^/m, "    ")
        file.puts(formatted)
        file.puts(data_feed_closing)
        formatted.size
      rescue => e
        raise e
      ensure
        file.close
      end
    end
def add_json ( name , json ) file = File . open ( name , "r+" ) begin pos_stack = [ 0 , 0 , 0 ] content_stack = [ "" , "" ] file . each do | line | unless file . eof? pos_stack . push ( file . pos ) pos_stack . shift content_stack . push ( line ) content_stack . shift end end file . seek ( pos_stack . first , IO :: SEEK_SET ) prev_content = content_stack . first . chomp! prev_content += "," unless prev_content [ - 1 ] == "[" file . puts ( "#{prev_content}" ) formatted = JSON . pretty_generate ( json ) . gsub ( / ^ /m , "    " ) file . puts ( formatted ) file . puts ( data_feed_closing ) formatted . size rescue => e raise e ensure file . close end end
NOTE: inefficient to open and close the file for every taxon... but
this allows us to see partial results sooner, and we don't mind the
additional pause (because our queries slow down the site):
NOTE : inefficient to open and close the file for every taxon ... but this allows us to see partial results sooner and we don ' t mind the additional pause ( because our queries slow down the site ) :
name
<not_specific>
json
<not_specific>