title: Stormcrawler: Apache Tika for parsing PDF properties
body: <p>I have added Tika as a reference to my StormCrawler implementation and that enables to fetch the PDF documents in the crawl. But, the <em>Title</em>, <em>Authors</em> and other properties don't get parsed. I have tried with different combinations to '<em>index.md.mapping</em>:' and added the corresponding properties to <em>ES_IndexInit</em>, but the content field in Kibana (index) for PDF's documents is always empty. Everything works for HTML pages. Can you please help with some pointers, if I am missing something or I can look at an example?</p>
  
  <hr>
  
  <p><strong><em>es-crawler.flux:</em></strong></p>
  
  <p><code> 
  name: "crawler"</p>
  
  <p>includes:
      - resource: true
        file: "/crawler-default.yaml"
        override: false</p>
  
  <pre><code>- resource: false
    file: "crawler-conf.yaml"
    override: true
  
  - resource: false
    file: "es-conf.yaml"
    override: true
  </code></pre>
  
  <p>spouts:
    - id: "spout"
      className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.AggregationSpout"
      parallelism: 10</p>
  
  <p>bolts:
    - id: "partitioner"
      className: "com.digitalpebble.stormcrawler.bolt.URLPartitionerBolt"
      parallelism: 1
    - id: "fetcher"
      className: "com.digitalpebble.stormcrawler.bolt.FetcherBolt"
      parallelism: 1
    - id: "sitemap"
      className: "com.digitalpebble.stormcrawler.bolt.SiteMapParserBolt"
      parallelism: 1
    - id: "parse"
      className: "com.digitalpebble.stormcrawler.bolt.JSoupParserBolt"
      parallelism: 5
    - id: "index"
      className: "com.digitalpebble.stormcrawler.elasticsearch.bolt.IndexerBolt"
      parallelism: 1
    - id: "status"
      className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
      parallelism: 1
    - id: "status_metrics"
      className: "com.digitalpebble.stormcrawler.elasticsearch.metrics.StatusMetricsBolt"
      parallelism: 4
    - id: "redirection_bolt"
      className: "com.digitalpebble.stormcrawler.tika.RedirectionBolt"
      parallelism: 1
    - id: "parser_bolt"
      className: "com.digitalpebble.stormcrawler.tika.ParserBolt"
      parallelism: 1</p>
  
  <p>streams:
    - from: "spout"
      to: "partitioner"
      grouping:
        type: SHUFFLE</p>
  
  <ul>
  <li><p>from: "spout"
  to: "status_metrics"
  grouping:
    type: SHUFFLE     </p></li>
  <li><p>from: "partitioner"
  to: "fetcher"
  grouping:
    type: FIELDS
    args: ["key"]</p></li>
  <li><p>from: "fetcher"
  to: "sitemap"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "sitemap"
  to: "parse"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "parse"
  to: "index"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "fetcher"
  to: "status"
  grouping:
    type: FIELDS
    args: ["url"]
    streamId: "status"</p></li>
  <li><p>from: "sitemap"
  to: "status"
  grouping:
    type: FIELDS
    args: ["url"]
    streamId: "status"</p></li>
  <li><p>from: "parse"
  to: "status"
  grouping:
    type: FIELDS
    args: ["url"]
    streamId: "status"</p></li>
  <li><p>from: "index"
  to: "status"
  grouping:
    type: FIELDS
    args: ["url"]
    streamId: "status"  </p></li>
  <li><p>from: "parse"
  to: "redirection_bolt"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "redirection_bolt"
  to: "parser_bolt"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "redirection_bolt"
  to: "index"
  grouping:
    type: LOCAL_OR_SHUFFLE</p></li>
  <li><p>from: "parser_bolt"
  to: "index"
  grouping:
    type: LOCAL_OR_SHUFFLE
  </code> </p></li>
  </ul>
  
  <p><strong><em>es-injector.flux:</em></strong>
  <code> 
  name: "injector"</p>
  
  <p>includes:
      - resource: true
        file: "/crawler-default.yaml"
        override: false</p>
  
  <pre><code>- resource: false
    file: "crawler-conf.yaml"
    override: true
  
  - resource: false
    file: "es-conf.yaml"
    override: true
  
  - resource: false
    file: "injection-conf.yaml"
    override: true
  </code></pre>
  
  <p>components:
    - id: "scheme"
      className: "com.digitalpebble.stormcrawler.util.StringTabScheme"
      constructorArgs:
        - DISCOVERED</p>
  
  <p>spouts:
    - id: "spout"
      className: "com.digitalpebble.stormcrawler.spout.FileSpout"
      parallelism: 1
      constructorArgs:
        - "."
        - "seeds.txt"
        - ref: "scheme"</p>
  
  <p>bolts:
    - id: "status"
      className: "com.digitalpebble.stormcrawler.elasticsearch.persistence.StatusUpdaterBolt"
      parallelism: 1
    - id: "parser_bolt"
      className: "com.digitalpebble.stormcrawler.tika.ParserBolt"
      parallelism: 1</p>
  
  <p>streams:
    - from: "spout"
      to: "status"
      grouping:
        type: FIELDS
        args: ["url"]
  </code> </p>
  
  <p><strong><em>pom.xml:</em></strong>
  <code> 
  http://maven.apache.org/maven-v4_0_0.xsd"></p>
  
  <pre><code>&lt;modelVersion&gt;4.0.0&lt;/modelVersion&gt;
  &lt;groupId&gt;xyz.com&lt;/groupId&gt;
  &lt;artifactId&gt;search&lt;/artifactId&gt;
  &lt;version&gt;search1.0&lt;/version&gt;
  &lt;packaging&gt;jar&lt;/packaging&gt;
  
  &lt;properties&gt;
      &lt;project.build.sourceEncoding&gt;UTF-8&lt;/project.build.sourceEncoding&gt;
  &lt;/properties&gt;
  
  &lt;build&gt;
      &lt;plugins&gt;
          &lt;plugin&gt;
              &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
              &lt;artifactId&gt;maven-compiler-plugin&lt;/artifactId&gt;
              &lt;version&gt;3.2&lt;/version&gt;
              &lt;configuration&gt;
                  &lt;source&gt;1.8&lt;/source&gt;
                  &lt;target&gt;1.8&lt;/target&gt;
              &lt;/configuration&gt;
          &lt;/plugin&gt;
          &lt;plugin&gt;
              &lt;groupId&gt;org.codehaus.mojo&lt;/groupId&gt;
              &lt;artifactId&gt;exec-maven-plugin&lt;/artifactId&gt;
              &lt;version&gt;1.3.2&lt;/version&gt;
              &lt;executions&gt;
                  &lt;execution&gt;
                      &lt;goals&gt;
                          &lt;goal&gt;exec&lt;/goal&gt;
                      &lt;/goals&gt;
                  &lt;/execution&gt;
              &lt;/executions&gt;
              &lt;configuration&gt;
                  &lt;executable&gt;java&lt;/executable&gt;
                  &lt;includeProjectDependencies&gt;true&lt;/includeProjectDependencies&gt;
                  &lt;includePluginDependencies&gt;false&lt;/includePluginDependencies&gt;
                  &lt;classpathScope&gt;compile&lt;/classpathScope&gt;
              &lt;/configuration&gt;
          &lt;/plugin&gt;
          &lt;plugin&gt;
              &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
              &lt;artifactId&gt;maven-shade-plugin&lt;/artifactId&gt;
              &lt;version&gt;1.3.3&lt;/version&gt;
              &lt;executions&gt;
                  &lt;execution&gt;
                      &lt;phase&gt;package&lt;/phase&gt;
                      &lt;goals&gt;
                          &lt;goal&gt;shade&lt;/goal&gt;
                      &lt;/goals&gt;
                      &lt;configuration&gt;
                          &lt;createDependencyReducedPom&gt;false&lt;/createDependencyReducedPom&gt;
                          &lt;transformers&gt;
                              &lt;transformer
                                  implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /&gt;
                              &lt;transformer
                                  implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"&gt;
                                &lt;mainClass&gt;org.apache.storm.flux.Flux&lt;/mainClass&gt;
                                &lt;manifestEntries&gt;
                                  &lt;Change&gt;&lt;/Change&gt;
                                  &lt;Build-Date&gt;&lt;/Build-Date&gt;
                                &lt;/manifestEntries&gt;
                              &lt;/transformer&gt;
                          &lt;/transformers&gt;
                          &lt;!-- The filters below are necessary if you want to include the Tika
                              module --&gt;
                          &lt;filters&gt;
                              &lt;filter&gt;
                                  &lt;artifact&gt;*:*&lt;/artifact&gt;
                                  &lt;excludes&gt;
                                      &lt;exclude&gt;META-INF/*.SF&lt;/exclude&gt;
                                      &lt;exclude&gt;META-INF/*.DSA&lt;/exclude&gt;
                                      &lt;exclude&gt;META-INF/*.RSA&lt;/exclude&gt;
                                  &lt;/excludes&gt;
                              &lt;/filter&gt;
                          &lt;/filters&gt;
                      &lt;/configuration&gt;
                  &lt;/execution&gt;
              &lt;/executions&gt;
          &lt;/plugin&gt;
      &lt;/plugins&gt;
  &lt;/build&gt;
  
  &lt;dependencies&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;org.apache.storm&lt;/groupId&gt;
          &lt;artifactId&gt;storm-core&lt;/artifactId&gt;
          &lt;version&gt;1.1.1&lt;/version&gt;
          &lt;scope&gt;provided&lt;/scope&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;org.apache.storm&lt;/groupId&gt;
          &lt;artifactId&gt;flux-core&lt;/artifactId&gt;
          &lt;version&gt;1.0.2&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;com.digitalpebble.stormcrawler&lt;/groupId&gt;
          &lt;artifactId&gt;storm-crawler-core&lt;/artifactId&gt;
          &lt;version&gt;1.7&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;com.digitalpebble.stormcrawler&lt;/groupId&gt;
          &lt;artifactId&gt;storm-crawler-elasticsearch&lt;/artifactId&gt;
          &lt;version&gt;1.7&lt;/version&gt;
      &lt;/dependency&gt;
      &lt;dependency&gt;
          &lt;groupId&gt;com.digitalpebble.stormcrawler&lt;/groupId&gt;
          &lt;artifactId&gt;storm-crawler-tika&lt;/artifactId&gt;
          &lt;version&gt;1.7&lt;/version&gt;
      &lt;/dependency&gt;
  &lt;/dependencies&gt;
  </code></pre>
  
  <p>
  </code> </p>