lib/metadata/extract.rb
# Extracts metadata from a file by guessing mimetype and calling matching
# extractor methods (which mostly call external programs to do their bidding.)
#
#   Metadata.extract('foo.png')
#
# Follows the Shared File Metadata Spec naming.
# http://wiki.freedesktop.org/wiki/Specifications/shared-filemetadata-spec
#
# There are a couple flags that control the behaviour of
# the metadata extractor:
#
#   Metadata.sha1sum = true # include File.SHA1Sum in the metadata
#   Metadata.md5sum = true  # include File.MD5Sum in the metadata
#   Metadata.include_name = true # include File.Name (file basename)
#   Metadata.include_path = true # include File.Path (file dirname)
#   Metadata.quiet = true   # override verbose to false
#   Metadata.verbose = true # print out status messages to stderr
#
# All strings are converted to UTF-8.
#
Extracts metadata from a file by guessing mimetype and calling matching
extractor methods (which mostly call external programs to do their bidding.)



Follows the Shared File Metadata Spec naming.

There are a couple flags that control the behaviour of
the metadata extractor.



All strings are converted to UTF-8.
Extracts metadata from a file by guessing mimetype and calling matching extractor methods ( which mostly call external programs to do their bidding . ) Follows the Shared File Metadata Spec naming . There are a couple flags that control the behaviour of the metadata extractor . All strings are converted to UTF - 8 .
def extract(filename, mimetype=MimeInfo.get(filename.to_s), charset=nil, pdf=nil)
    verbose = verbose && !quiet
    filename = filename.to_s
    mimetype = Mimetype[mimetype] unless mimetype.is_a?( Mimetype )
    unless File.exist?(filename)
      rv = {}
      if self.include_name
        rv['File.Name'] = enc_utf8(File.basename(filename), nil)
      end
      if self.include_path
        rv['File.Path'] = enc_utf8(File.dirname(filename), nil)
      end
      rv['File.Format'] ||= mimetype.to_s
      return rv
    end
    mts = mimetype.ancestors
    mt = mts.shift
    rv = nil
    new_methods = public_methods(false)
    STDERR.puts "Processing #{filename}", " Metadata extraction" if verbose
    while mt.is_a?(Mimetype) and mt != Mimetype
      STDERR.puts "  Trying #{mt}" if verbose
      mn = mt.to_s.gsub(/[^a-z0-9]/i,"_").to_sym
      if new_methods.include?( mn )
        begin
          rv = __send__( mn, filename, charset )
          STDERR.puts "  OK" if verbose
          break
        rescue => e
          STDERR.puts(e, e.message, e.backtrace) if verbose
        end
      end
      mt = mts.shift
    end
    unless rv
      STDERR.puts "  Falling back to extract" if verbose
      rv = extract_extract_info(filename)
    end
    if self.include_name
      rv['File.Name'] = enc_utf8(File.basename(filename), nil)
    end
    if self.include_path
      rv['File.Path'] = enc_utf8(File.dirname(filename), nil)
    end
    rv['File.Format'] ||= mimetype.to_s
    if File.file?(filename)
      if self.sha1sum
        secure_filename(filename){|sfn|
          rv['File.SHA1Sum'] = `sha1sum #{sfn}`.split(" ",2)[0]
        }
      end
      if self.md5sum
        secure_filename(filename){|sfn|
          rv['File.MD5Sum'] = `md5sum #{sfn}`.split(" ",2)[0]
        }
      end
    end
    rv['File.Size'] = (
      if File.directory?(filename)
        Dir.entries(filename).size-2
      else
        File.size(filename)
      end)
    rv['File.Content'] = extract_text(filename, mimetype, charset, false) unless Metadata.no_text
    pdf ||= filename + "-temp.pdf"
    if File.exist?(pdf)
      pdf_metadata = application_pdf(pdf, charset)
      overrides = %w(Image.DimensionUnit Image.Width Image.Height Doc.PageCount
                     Doc.PageSizeName)
      optrides = %w(Doc.WordCount Doc.Title Doc.Author)
      overrides.each{|o| rv[o] = pdf_metadata[o] }
      optrides.each {|o| rv[o] ||= pdf_metadata[o] }
      if !Metadata.no_text and not to_s =~ /postscript/
        rv['File.Content'] = extract_text(pdf, Mimetype['application/pdf'], charset, false)
      end
    end
    if guess_title or guess_metadata or guess_pubdata
      gem_require 'metadata/title_guesser'
      gem_require 'metadata/publication_guesser'
      gem_require 'metadata/reference_guesser'
      text = (rv['File.Content'] || extract_text(filename, mimetype, charset, false))
      guess = extract_guesses(text)
      if guess['Doc.Title'] and (rv['Doc.Title'].nil? or rv['Doc.Title'] =~ /(^[a-z])|((\.(dvi|doc)|WORD)$)|^Slide 1$|^PowerPoint Presentation$/)
        rv['Doc.Title'] = guess['Doc.Title']
      end
    end
    if use_citeseer and rv['Doc.Title'] and mimetype.to_s =~ /pdf|postscript|dvi|tex/
      rv.merge!(citeseer_extract(rv['Doc.Title']))
    end
    if use_dblp and rv['Doc.Title'] and mimetype.to_s =~ /pdf|postscript|dvi|tex/
      rv.merge!(dblp_extract(rv['Doc.Title']))
    end
    if guess_metadata or guess_pubdata
      %w(Doc.Publisher Doc.Published Doc.Publication Doc.Genre Event.Name Event.Organizer
      ).each{|field|
        rv[field] ||= guess[field] if guess[field]
      }
    end
    if guess_metadata
      %w(Doc.Citations Doc.Description Doc.ACMCategories Doc.Keywords
      ).each{|field|
        rv[field] ||= guess[field] if guess[field]
      }
    end
    rv['File.Modified'] = parse_time(File.mtime(filename.to_s).iso8601)
    rv.delete_if{|k,v| v.nil? }
    rv
  end
def extract ( filename , mimetype = MimeInfo . get ( filename . to_s ) , charset = nil , pdf = nil ) verbose = verbose && ! quiet filename = filename . to_s mimetype = Mimetype [ mimetype ] unless mimetype . is_a? ( Mimetype ) unless File . exist? ( filename ) rv = { } if self . include_name rv [ 'File.Name' ] = enc_utf8 ( File . basename ( filename ) , nil ) end if self . include_path rv [ 'File.Path' ] = enc_utf8 ( File . dirname ( filename ) , nil ) end rv [ 'File.Format' ] ||= mimetype . to_s return rv end mts = mimetype . ancestors mt = mts . shift rv = nil new_methods = public_methods ( false ) STDERR . puts "Processing #{filename}" , " Metadata extraction" if verbose while mt . is_a? ( Mimetype ) and mt != Mimetype STDERR . puts "  Trying #{mt}" if verbose mn = mt . to_s . gsub ( / [^a-z0-9] /i , "_" ) . to_sym if new_methods . include? ( mn ) begin rv = __send__ ( mn , filename , charset ) STDERR . puts "  OK" if verbose break rescue => e STDERR . puts ( e , e . message , e . backtrace ) if verbose end end mt = mts . shift end unless rv STDERR . puts "  Falling back to extract" if verbose rv = extract_extract_info ( filename ) end if self . include_name rv [ 'File.Name' ] = enc_utf8 ( File . basename ( filename ) , nil ) end if self . include_path rv [ 'File.Path' ] = enc_utf8 ( File . dirname ( filename ) , nil ) end rv [ 'File.Format' ] ||= mimetype . to_s if File . file? ( filename ) if self . sha1sum secure_filename ( filename ) { | sfn | rv [ 'File.SHA1Sum' ] = ` sha1sum  #{ sfn } ` . split ( " " , 2 ) [ 0 ] } end if self . md5sum secure_filename ( filename ) { | sfn | rv [ 'File.MD5Sum' ] = ` md5sum  #{ sfn } ` . split ( " " , 2 ) [ 0 ] } end end rv [ 'File.Size' ] = ( if File . directory? ( filename ) Dir . entries ( filename ) . size - 2 else File . size ( filename ) end ) rv [ 'File.Content' ] = extract_text ( filename , mimetype , charset , false ) unless Metadata . no_text pdf ||= filename + "-temp.pdf" if File . exist? ( pdf ) pdf_metadata = application_pdf ( pdf , charset ) overrides = %w( Image.DimensionUnit Image.Width Image.Height Doc.PageCount Doc.PageSizeName ) optrides = %w( Doc.WordCount Doc.Title Doc.Author ) overrides . each { | o | rv [ o ] = pdf_metadata [ o ] } optrides . each { | o | rv [ o ] ||= pdf_metadata [ o ] } if ! Metadata . no_text and not to_s =~ / postscript / rv [ 'File.Content' ] = extract_text ( pdf , Mimetype [ 'application/pdf' ] , charset , false ) end end if guess_title or guess_metadata or guess_pubdata gem_require 'metadata/title_guesser' gem_require 'metadata/publication_guesser' gem_require 'metadata/reference_guesser' text = ( rv [ 'File.Content' ] || extract_text ( filename , mimetype , charset , false ) ) guess = extract_guesses ( text ) if guess [ 'Doc.Title' ] and ( rv [ 'Doc.Title' ] . nil? or rv [ 'Doc.Title' ] =~ / (^[a-z])|(( \. (dvi|doc)|WORD)$)|^Slide 1$|^PowerPoint Presentation$ / ) rv [ 'Doc.Title' ] = guess [ 'Doc.Title' ] end end if use_citeseer and rv [ 'Doc.Title' ] and mimetype . to_s =~ / pdf|postscript|dvi|tex / rv . merge! ( citeseer_extract ( rv [ 'Doc.Title' ] ) ) end if use_dblp and rv [ 'Doc.Title' ] and mimetype . to_s =~ / pdf|postscript|dvi|tex / rv . merge! ( dblp_extract ( rv [ 'Doc.Title' ] ) ) end if guess_metadata or guess_pubdata %w( Doc.Publisher Doc.Published Doc.Publication Doc.Genre Event.Name Event.Organizer ) . each { | field | rv [ field ] ||= guess [ field ] if guess [ field ] } end if guess_metadata %w( Doc.Citations Doc.Description Doc.ACMCategories Doc.Keywords ) . each { | field | rv [ field ] ||= guess [ field ] if guess [ field ] } end rv [ 'File.Modified' ] = parse_time ( File . mtime ( filename . to_s ) . iso8601 ) rv . delete_if { | k , v | v . nil? } rv end
Extracts metadata from a file by guessing mimetype and calling matching
extractor methods (which mostly call external programs to do their bidding.)
Extracts metadata from a file by guessing mimetype and calling matching extractor methods ( which mostly call external programs to do their bidding . )
filename
<not_specific>
mimetype
<not_specific>
get
<not_specific>
to_s
<not_specific>
charset
<not_specific>
pdf
<not_specific>