title: How to save files found with wildcard file & folder to the right file name on the central server with rsyslog?
body: <p>I am trying to use rainerscript and rsyslog v8.38 to grab logs off my servers by both wildcard and wildcard files in wildcard folders, and then save them in the same folder structure at the far end. I have wildcards working thanks to the article <a href="https://selivan.github.io/2017/02/07/rsyslog-log-forward-save-filename-handle-multi-line-failover.html" rel="nofollow noreferrer">here</a>, but I'm trying to extend this concept to work for the wildcard folders as well.</p>
  
  <p>Currently I collect the files from the folders correctly, but once it's saved the lines from all the files are saved in to one file for each folder, named the same as the folder is, for example, if I do this on my server:</p>
  
  <pre><code>echo "TEST1" &gt;&gt; /srv/log/test-new/test.log
  echo "TEST1" &gt;&gt; /srv/log/test-new/test-new.log
  </code></pre>
  
  <p>I end up with this on my central server:</p>
  
  <pre><code># cat /srv/rsyslog/2018/HOSTNAME/10/26/test-new
  
  &lt;133&gt;2018-10-26T15:32:37.975449+00:00 HOSTNAME  test-new/test.log nested-srv-logs TEST1
  &lt;133&gt;2018-10-26T15:32:51.042633+00:00 HOSTNAME  test-new/test-new.log nested-srv-logs TEST1
  </code></pre>
  
  <p>I was hoping that I could have the files saved on the central server in the same folder structure as they were found on the sending machine, is this possible?</p>
  
  <p>My config is as below, sending machine has:</p>
  
  <pre><code>module(load="omrelp")
  module(load="omfwd")
  
  template(name="CustomForwardFormat" type="list") {
      constant(value="&lt;")
      property(name="pri")
      constant(value="&gt;")
      property(name="timestamp" dateFormat="rfc3339")
      constant(value=" ")
      property(name="hostname")
      constant(value=" ")
      property(name=".suffix")
      constant(value=" ")
      property(name="syslogtag" position.from="1" position.to="32")
      property(name="msg" spifno1stsp="on" )
      property(name="msg")
      constant(value="\n")
      }
  
  ruleset(name="sendToLogserver") {
      action(type="omrelp" Target="rsyslog" Port="25014" template="CustomForwardFormat"
      queue.type="LinkedList" queue.size="10000" queue.filename="q_sendToLogserver" queue.highwatermark="9000"
      queue.lowwatermark="50" queue.maxdiskspace="500m" queue.saveonshutdown="on" action.resumeRetryCount="-1"
      action.reportSuspension="on" action.reportSuspensionContinuation="on" action.resumeInterval="10")
  }
  
  ruleset(name="sendToJsonLogserver") {
      action(type="omfwd" protocol="tcp" Target="logstash" Port="5114" template="RSYSLOG_SyslogProtocol23Format"
      queue.type="LinkedList" queue.size="10000" queue.filename="q_sendToJsonLogserver" queue.highwatermark="9000"
      queue.lowwatermark="50" queue.maxdiskspace="500m" queue.saveonshutdown="on" action.resumeRetryCount="-1"
      action.reportSuspension="on" action.reportSuspensionContinuation="on" action.resumeInterval="10")
  }
  
  input(type="imfile"
      File="/srv/log/*.log"
          Tag="srv-logs"
          Ruleset="srv_logs"
          addMetadata="on")
  
  input(type="imfile"
      File="/srv/log/*/*.log"
          Tag="nested-srv-logs"
          Ruleset="srv_logs"
          addMetadata="on")
  
  ruleset(name="srv_logs") {
      # http://www.rsyslog.com/doc/v8-stable/rainerscript/functions.html
            # re_extract(expr, re, match, submatch, no-found)
      set $.suffix=re_extract($!metadata!filename, "(.*)/([^/]*)", 0, 2, "unknown.log");
      if ( $programname == "nested-srv-logs" ) then {
        set $.sub-suffix=re_extract($!metadata!filename, "(.*)/([^/]*)/(.*)", 0, 2, "unknown.log");
          set $.suffix=$.sub-suffix &amp; "/" &amp; $.suffix;
      }
      if( $!metadata!filename contains 'json' ) then {
              call sendToJsonLogserver
      } else {
        call sendToLogserver
      }
      stop
  }
  </code></pre>
  
  <p>Central server has:</p>
  
  <pre><code>module(load="imrelp")
  input(type="imrelp" port="25014" ruleset="RemoteLogProcess")
  
  module(load="builtin:omfile" FileOwner="syslog" FileGroup="syslog" dirOwner="syslog" dirGroup="syslog" FileCreateMode="0644" DirCreateMode="0755")
  
  template(name="CustomForwardFormat" type="list") {
      constant(value="&lt;")
      property(name="pri")
      constant(value="&gt;")
      property(name="timestamp" dateFormat="rfc3339")
      constant(value=" ")
      property(name="hostname")
      constant(value=" ")
      property(name=".suffix")
      constant(value=" ")
      property(name="syslogtag" position.from="1" position.to="32")
      property(name="msg" spifno1stsp="on" )
      property(name="msg")
      constant(value="\n")
      }
  
  $EscapeControlCharactersOnReceive off
  
  template(name="FloowLogSavePath" type="list") {
      constant(value="/srv/rsyslog/")
      property(name="timegenerated" dateFormat="year")
      constant(value="/")
      property(name="hostname")
      constant(value="/")
      property(name="timegenerated" dateFormat="month")
      constant(value="/")
      property(name="timegenerated" dateFormat="day")
      constant(value="/")
      property(name="$.logpath" )
  }
  
  ruleset(name="RemoteLogProcess") {
      # For facilities local0-7 set log filename from $programname field: replace __ with /
      if ( $syslogfacility &gt;= 16 ) then
      {
          set $.logpath = replace($programname, "__", "/");
          action(type="omfile" dynaFileCacheSize="1024" dynaFile="FloowLogSavePath" template="CustomForwardFormat"
          flushOnTXEnd="off" asyncWriting="on" flushInterval="1" ioBufferSize="64k")
      } else {
          if (($syslogfacility == 0)) then {
              set $.logpath = "kern.log";
          } else if (($syslogfacility == 1)) then {
              set $.logpath = "user";
          } else if (($syslogfacility == 2)) then {
              set $.logpath = "mail";
          } else if (($syslogfacility == 3)) then {
              set $.logpath = "daemon";
          } else if (($syslogfacility == 4) or ($syslogfacility == 10)) then {
              set $.logpath = "auth.log";
          } else if (($syslogfacility == 9) or ($syslogfacility == 15)) then {
              set $.logpath = "cron";
          } else {
              set $.logpath ="syslog";
          }
          # Built-in template RSYSLOG_FileFormat: High-precision timestamps and timezone information
          action(type="omfile" dynaFileCacheSize="1024" dynaFile="FloowLogSavePath" template="CustomForwardFormat"
          flushOnTXEnd="off" asyncWriting="on" flushInterval="1" ioBufferSize="64k")
      }
  }
  </code></pre>