Add multiline RFC5424 support

Signed-off-by: Mat Monroe (onet) <mmonroe@aurora.tech>
fluent · Jan 22, 2020 · 8eeb872 · 8eeb872
1 parent 8610cd7
commit 8eeb872
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 2 deletions.
diff --git a/lib/fluent/plugin/parser_syslog.rb b/lib/fluent/plugin/parser_syslog.rb
@@ -27,8 +27,8 @@ class SyslogParser < Parser
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
-      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424 = /\A(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
+      REGEXP_RFC5424_WITH_PRI = /\A\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
@@ -141,6 +141,9 @@ def parse_plain(text, &block)
                 end
               end
               record[name] = value if @keep_time_key
+            when "message"
+              value.chomp!
+              record[name] = value
             else
               record[name] = value
             end

diff --git a/test/plugin/test_parser_syslog.rb b/test/plugin/test_parser_syslog.rb
@@ -188,6 +188,42 @@ def test_parse_with_rfc5424_message
                    @parser.instance.patterns['format'])
     end
 
+    def test_parse_with_rfc5424_message_trailing_eol
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!\n"
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+    end
+
+    def test_parse_with_rfc5424_multiline_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        'with_priority' => true,
+                        )
+      text = "<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi,\nfrom\nFluentd!"
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi,\nfrom\nFluentd!", record["message"]
+      end
+      assert_equal(Fluent::Plugin::SyslogParser::REGEXP_RFC5424_WITH_PRI,
+                   @parser.instance.patterns['format'])
+    end
+
     def test_parse_with_rfc5424_message_and_without_priority
       @parser.configure(
                         'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',