app/controllers/discourse-media-wiki-import/import_controller.rb
##
# Parses the uploaded MediaWiki export file and imports each page under the chosen Discourse cateogory.
#
Parses the uploaded MediaWiki export file and imports each page under the chosen Discourse cateogory.
Parses the uploaded MediaWiki export file and imports each page under the chosen Discourse cateogory .
def import
      params.require([:uploadedMediaWikiExportUrl, :categoryId])
      start = Time.now
      upload = Upload.get_from_url(params[:uploadedMediaWikiExportUrl])
      local_path = Discourse.store.path_for(upload)
      processed_pages = 0
      new_pages = 0
      updated_pages = 0
      failed_pages = 0
      mwns = 'http://www.mediawiki.org/xml/export-0.10/'
      xml_doc = Nokogiri::XML(File.open(local_path))
      site_url = URI.parse(xml_doc.xpath('//mediawiki:siteinfo', 'mediawiki'=>mwns).first.at('base').text)
      site_domain = "#{site_url.scheme}://#{site_url.host}/"
      pages = xml_doc.xpath('//mediawiki:page', 'mediawiki'=>mwns).reverse
      wiki_templates = Hash.new
      wiki_page_topic_map = Hash.new
      pages.each do |page|
        begin
          wiki_page_id = page.at('id').text
          wiki_page_title = page.at('title').text
          if wiki_page_title.start_with?('Template:')
            unless wiki_page_title.include?('TOC')
              template_name = wiki_page_title.delete_prefix('Template:')
              template_content = page.at('revision').at('text').text
              wiki_templates[template_name] = template_content
            end
          else
            topic = find_or_create_topic(
                wiki_page_id: wiki_page_id, 
                category_id: params[:categoryId]
              )
            wiki_page_topic_map[wiki_page_title] = topic
          end
        rescue => exception
            Rails.logger.error("Failed to convert MediaWiki page #{wiki_page_id}: #{exception.message}\n#{exception.backtrace.join("\n")}")
            failed_pages += 1
        end
      end
      pages.each do |page|
        processed_pages += 1
        begin
          wiki_page_id = page.at('id').text
          wiki_page_title = page.at('title').text
          wiki_page_text = page.at('revision').at('text').text
          if wiki_page_title.start_with?('Template:')
            processed_pages -= 1
            next
          end
          topic = wiki_page_topic_map[wiki_page_title]
          is_new_topic = topic.title.blank?
          wiki_page_text = convert_mediawiki_to_html(wiki_page_text: wiki_page_text, wiki_page_topic_map: wiki_page_topic_map, site_domain: site_domain, templates: wiki_templates)
          update_topic_content(topic: topic, wiki_page_title: wiki_page_title, wiki_page_text: wiki_page_text)
          if is_new_topic
            new_pages += 1
          else
            updated_pages += 1
          end
        rescue => exception
            Rails.logger.error("Failed to convert MediaWiki page #{wiki_page_id}: #{exception.message}\n#{exception.backtrace.join("\n")}")
            failed_pages += 1
        end
      end
      finish = Time.now
      duration = finish - start
      render_json_dump("Successfully imported #{processed_pages} MediaWiki articles (#{new_pages} new articles, #{updated_pages} updated articles, #{failed_pages} failed articles) - #{duration.to_i} seconds")
    end
def import params . require ( [ :uploadedMediaWikiExportUrl , :categoryId ] ) start = Time . now upload = Upload . get_from_url ( params [ :uploadedMediaWikiExportUrl ] ) local_path = Discourse . store . path_for ( upload ) processed_pages = 0 new_pages = 0 updated_pages = 0 failed_pages = 0 mwns = 'http://www.mediawiki.org/xml/export-0.10/' xml_doc = Nokogiri :: XML ( File . open ( local_path ) ) site_url = URI . parse ( xml_doc . xpath ( '//mediawiki:siteinfo' , 'mediawiki' => mwns ) . first . at ( 'base' ) . text ) site_domain = "#{site_url.scheme}://#{site_url.host}/" pages = xml_doc . xpath ( '//mediawiki:page' , 'mediawiki' => mwns ) . reverse wiki_templates = Hash . new wiki_page_topic_map = Hash . new pages . each do | page | begin wiki_page_id = page . at ( 'id' ) . text wiki_page_title = page . at ( 'title' ) . text if wiki_page_title . start_with? ( 'Template:' ) unless wiki_page_title . include? ( 'TOC' ) template_name = wiki_page_title . delete_prefix ( 'Template:' ) template_content = page . at ( 'revision' ) . at ( 'text' ) . text wiki_templates [ template_name ] = template_content end else topic = find_or_create_topic ( wiki_page_id : wiki_page_id , category_id : params [ :categoryId ] ) wiki_page_topic_map [ wiki_page_title ] = topic end rescue => exception Rails . logger . error ( "Failed to convert MediaWiki page #{wiki_page_id}: #{exception.message}\n#{exception.backtrace.join("\n")}" ) failed_pages += 1 end end pages . each do | page | processed_pages += 1 begin wiki_page_id = page . at ( 'id' ) . text wiki_page_title = page . at ( 'title' ) . text wiki_page_text = page . at ( 'revision' ) . at ( 'text' ) . text if wiki_page_title . start_with? ( 'Template:' ) processed_pages -= 1 next end topic = wiki_page_topic_map [ wiki_page_title ] is_new_topic = topic . title . blank? wiki_page_text = convert_mediawiki_to_html ( wiki_page_text : wiki_page_text , wiki_page_topic_map : wiki_page_topic_map , site_domain : site_domain , templates : wiki_templates ) update_topic_content ( topic : topic , wiki_page_title : wiki_page_title , wiki_page_text : wiki_page_text ) if is_new_topic new_pages += 1 else updated_pages += 1 end rescue => exception Rails . logger . error ( "Failed to convert MediaWiki page #{wiki_page_id}: #{exception.message}\n#{exception.backtrace.join("\n")}" ) failed_pages += 1 end end finish = Time . now duration = finish - start render_json_dump ( "Successfully imported #{processed_pages} MediaWiki articles (#{new_pages} new articles, #{updated_pages} updated articles, #{failed_pages} failed articles) - #{duration.to_i} seconds" ) end
Parses the uploaded MediaWiki export file and imports each page under the chosen Discourse cateogory.
Parses the uploaded MediaWiki export file and imports each page under the chosen Discourse cateogory .
# first pass: find or create topics # get XML element values needed for import # skip TOC # store template content # store topic in the mapping for the second pass # second pass: update topic content # get XML element values needed for import # ignore any template pages as they aren't content # increase counter for new or updated pages