Provide CRLF newline style choice on formatter plugins. Fix #3151

lib/fluent/plugin/formatter_hash.rb

@@ -22,10 +22,21 @@ class HashFormatter < Formatter
       Plugin.register_formatter('hash', self)
 
       config_param :add_newline, :bool, default: true
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
+
+      def configure(conf)
+        super
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
+      end
 
       def format(tag, time, record)
         line = record.to_s
-        line << "\n".freeze if @add_newline
+        line << @newline.freeze if @add_newline
         line
       end
     end

lib/fluent/plugin/formatter_json.rb

@@ -24,6 +24,7 @@ class JSONFormatter < Formatter
 
       config_param :json_parser, :string, default: 'oj'
       config_param :add_newline, :bool, default: true
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
 
       def configure(conf)
         super
@@ -41,10 +42,17 @@ def configure(conf)
         unless @add_newline
           define_singleton_method(:format, method(:format_without_nl))
         end
+
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}\n"
+        "#{@dump_proc.call(record)}#{@newline}"
       end
 
       def format_without_nl(tag, time, record)

lib/fluent/plugin/formatter_ltsv.rb

@@ -26,6 +26,18 @@ class LabeledTSVFormatter < Formatter
       config_param :delimiter, :string, default: "\t"
       config_param :label_delimiter, :string, default: ":"
       config_param :add_newline, :bool, default: true
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
+
+      def configure(conf)
+        super
+
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
+      end
 
       # TODO: escaping for \t in values
       def format(tag, time, record)
@@ -34,7 +46,7 @@ def format(tag, time, record)
           formatted << @delimiter if formatted.length.nonzero?
           formatted << "#{label}#{@label_delimiter}#{value}"
         end
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline.freeze if @add_newline
         formatted
       end
     end

lib/fluent/plugin/formatter_out_file.rb

@@ -32,19 +32,27 @@ class OutFileFormatter < Formatter
         else "\t"
         end
       end
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
       config_set_default :time_type, :string
       config_set_default :time_format, nil # time_format nil => iso8601
 
       def configure(conf)
         super
         @timef = time_formatter_create
+
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
       end
 
       def format(tag, time, record)
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
+        "#{header}#{Yajl.dump(record)}#{@newline}"
       end
     end
   end

lib/fluent/plugin/formatter_single_value.rb

@@ -23,10 +23,22 @@ class SingleValueFormatter < Formatter
 
       config_param :message_key, :string, default: 'message'
       config_param :add_newline, :bool, default: true
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
+
+      def configure(conf)
+        super
+
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
+      end
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
+        text << @newline.freeze if @add_newline
         text
       end
     end

lib/fluent/plugin/formatter_tsv.rb

@@ -27,10 +27,23 @@ class TSVFormatter < Formatter
       config_param :delimiter, :string, default: "\t"
       desc 'The parameter to enable writing to new lines'
       config_param :add_newline, :bool, default: true
+      desc 'The newline code'
+      config_param :newline, :enum, list: [:lf, :crlf], default: :lf
+
+      def configure(conf)
+        super
+
+        @newline = case newline
+                   when :lf
+                     "\n"
+                   when :crlf
+                     "\r\n"
+                   end
+      end
 
       def format(tag, time, record)
         formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline.freeze if @add_newline
         formatted
       end
     end

test/plugin/test_formatter_hash.rb

@@ -19,11 +19,14 @@ def record
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}#{newline}!, formatted.encode(Encoding::UTF_8))
   end
 
   def test_format_without_newline

test/plugin/test_formatter_json.rb

@@ -25,12 +25,17 @@ def symbolic_record
     {:message => :awesome}
   end
 
-  data('oj' => 'oj', 'yajl' => 'yajl')
+  data('oj with LF' => ['oj', "lf", "\n"],
+       'oj with CRLF' => ['oj', "crlf", "\r\n"],
+       'yajl with LF' => ['yajl', "lf", "\n"],
+       'yajl with CRLF' => ['yajl', "crlf", "\r\n"]
+      )
   def test_format(data)
-    d = create_driver('json_parser' => data)
+    parser, newline_conf, newline = data
+    d = create_driver('json_parser' => parser, 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{newline}", formatted)
   end
 
   data('oj' => 'oj', 'yajl' => 'yajl')

test/plugin/test_formatter_ltsv.rb

@@ -36,11 +36,14 @@ def test_config_params
     assert_equal  false, d.instance.add_newline
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message:awesome\tgreeting:hello\n", formatted)
+    assert_equal("message:awesome\tgreeting:hello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -50,13 +53,18 @@ def test_format_without_newline
     assert_equal("message:awesome\tgreeting:hello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
+
     d = create_driver(
       'delimiter'       => ',',
       'label_delimiter' => '=',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("message=awesome,greeting=hello\n", formatted)
+    assert_equal("message=awesome,greeting=hello#{newline}", formatted)
   end
 end

test/plugin/test_formatter_out_file.rb

@@ -54,42 +54,58 @@ def test_configured_with_utc_or_localtime(data)
     end
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time
-    d = create_driver('output_time' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time(data)
+    newline_conf, newline = data
+    d = create_driver('output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{tag}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_tag
-    d = create_driver('output_tag' => 'false')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_tag(data)
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}#{newline}", formatted)
   end
 
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
   def test_format_without_time_and_tag
-    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    newline_conf, newline = data
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false', 'newline' => newline_conf)
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 
-  def test_format_without_time_and_tag_against_string_literal_configure
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_without_time_and_tag_against_string_literal_configure(data)
+    newline_conf, newline = data
     d = create_driver(%[
                         utc         true
                         output_tag  false
                         output_time false
+                        newline     #{newline_conf}
                       ])
     formatted = d.instance.format('tag', @time, record)
 
-    assert_equal("#{Yajl.dump(record)}\n", formatted)
+    assert_equal("#{Yajl.dump(record)}#{newline}", formatted)
   end
 end

test/plugin/test_formatter_single_value.rb

@@ -17,10 +17,13 @@ def test_config_params_message_key
     assert_equal "foobar", d.instance.message_key
   end
 
-  def test_format
-    d = create_driver
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver('newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
-    assert_equal("awesome\n", formatted)
+    assert_equal("awesome#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -29,10 +32,13 @@ def test_format_without_newline
     assert_equal("awesome", formatted)
   end
 
-  def test_format_with_message_key
-    d = create_driver('message_key' => 'foobar')
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_message_key(data)
+    newline_conf, newline = data
+    d = create_driver('message_key' => 'foobar', 'newline' => newline_conf)
     formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
 
-    assert_equal("foo\n", formatted)
+    assert_equal("foo#{newline}", formatted)
   end
 end

test/plugin/test_formatter_tsv.rb

@@ -37,13 +37,17 @@ def test_config_params
     assert_equal false, d.instance.add_newline
   end
 
-  def test_format
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
+      'newline' => newline_conf
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome\thello\n", formatted)
+    assert_equal("awesome\thello#{newline}", formatted)
   end
 
   def test_format_without_newline
@@ -56,13 +60,17 @@ def test_format_without_newline
     assert_equal("awesome\thello", formatted)
   end
 
-  def test_format_with_customized_delimiters
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format_with_customized_delimiters(data)
+    newline_conf, newline = data
     d = create_driver(
       'keys' => 'message,greeting',
       'delimiter' => ',',
+      'newline' => newline_conf,
     )
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("awesome,hello\n", formatted)
+    assert_equal("awesome,hello#{newline}", formatted)
   end
 end

test/test_formatter.rb

@@ -271,6 +271,7 @@ def test_config_params
 
     def test_format
       formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome\n", formatted)
     end