Skip to content

Commit

Permalink
Follow newline code changes
Browse files Browse the repository at this point in the history 
Signed-off-by: Hiroshi Hatake <hatake@clear-code.com>
  • Loading branch information
@cosmo0920
cosmo0920 committed Oct 22, 2020
1 parent fdcfaf4 commit 73b4fd8
Show file tree
Hide file tree
Showing 8 changed files with 109 additions and 40 deletions.
28 changes: 22 additions & 6 deletions test/command/test_binlog_reader.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,14 @@ def timezone(timezone = 'UTC')
end

class TestHead < TestBaseCommand
setup do
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

sub_test_case 'initialize' do
data(
'file is not passed' => %w(),
Expand Down Expand Up @@ -138,7 +146,7 @@ class TestHead < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
head = BinlogReaderCommand::Head.new(argv)
out = capture_stdout { head.call }
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
end
end

Expand All @@ -149,7 +157,7 @@ class TestHead < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
head = BinlogReaderCommand::Head.new(argv)
out = capture_stdout { head.call }
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
end
end

Expand All @@ -169,7 +177,7 @@ class TestHead < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
head = BinlogReaderCommand::Head.new(argv)
out = capture_stdout { head.call }
assert_equal "#{Yajl.dump(@record)}\n", out
assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
end
end

Expand Down Expand Up @@ -198,6 +206,14 @@ class TestHead < TestBaseCommand
end

class TestCat < TestBaseCommand
setup do
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

sub_test_case 'initialize' do
data(
'file is not passed' => [],
Expand Down Expand Up @@ -254,7 +270,7 @@ class TestCat < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
head = BinlogReaderCommand::Cat.new(argv)
out = capture_stdout { head.call }
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
end
end

Expand All @@ -265,7 +281,7 @@ class TestCat < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
head = BinlogReaderCommand::Cat.new(argv)
out = capture_stdout { head.call }
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
end
end

Expand All @@ -276,7 +292,7 @@ class TestCat < TestBaseCommand
create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
head = BinlogReaderCommand::Cat.new(argv)
out = capture_stdout { head.call }
assert_equal "#{Yajl.dump(@record)}\n", out
assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
end
end

Expand Down
7 changes: 6 additions & 1 deletion test/plugin/test_filter_stdout.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ def setup
@old_tz = ENV["TZ"]
ENV["TZ"] = "UTC"
Timecop.freeze
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

def teardown
Expand Down Expand Up @@ -106,7 +111,7 @@ def test_include_time_key
def test_format_json
d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
assert_equal "{\"test\":\"test\"}\n", out
assert_equal "{\"test\":\"test\"}#{@default_newline}", out
end
end

Expand Down
7 changes: 6 additions & 1 deletion test/plugin/test_formatter_json.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ class JsonFormatterTest < ::Test::Unit::TestCase

def setup
@time = event_time
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

def create_driver(conf = "")
Expand Down Expand Up @@ -51,6 +56,6 @@ def test_format_with_symbolic_record(data)
d = create_driver('json_parser' => data)
formatted = d.instance.format(tag, @time, symbolic_record)

assert_equal("#{JSON.generate(record)}\n", formatted)
assert_equal("#{JSON.generate(record)}#{@default_newline}", formatted)
end
end
7 changes: 6 additions & 1 deletion test/plugin/test_formatter_out_file.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
class OutFileFormatterTest < ::Test::Unit::TestCase
def setup
@time = event_time
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

def create_driver(conf = {})
Expand Down Expand Up @@ -48,7 +53,7 @@ def test_configured_with_utc_or_localtime(data)
oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
d = create_driver(config_element('ROOT', '', {key => value}))
tag = 'test'
assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}#{@default_newline}", d.instance.format(tag, time, record)
ensure
ENV['TZ'] = oldtz
end
Expand Down
41 changes: 23 additions & 18 deletions test/plugin/test_out_file.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ def setup
Fluent::Test.setup
FileUtils.rm_rf(TMP_DIR)
FileUtils.mkdir_p(TMP_DIR)
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
Expand Down Expand Up @@ -91,7 +96,7 @@ def create_driver(conf = CONFIG, opts = {})
end
end
assert_equal 1, d.formatted.size
assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n], d.formatted[0]
assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
end

test 'no configuration error raised for basic configuration using "*" (v0.12 style)' do
Expand Down Expand Up @@ -296,11 +301,11 @@ def create_driver(conf = CONFIG, opts = {})

assert_equal 5, d.formatted.size

r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}\n!
r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}\n!
r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}\n!
r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}\n!
r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}\n!
r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}#{@default_newline}!
r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}#{@default_newline}!
r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}#{@default_newline}!
r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}#{@default_newline}!
r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}#{@default_newline}!
assert_equal r1, d.formatted[0]
assert_equal r2, d.formatted[1]
assert_equal r3, d.formatted[2]
Expand Down Expand Up @@ -329,8 +334,8 @@ def create_driver(conf = CONFIG, opts = {})
d.feed(time, {"a"=>2})
end
assert_equal 2, d.formatted.size
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n], d.formatted[0]
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n], d.formatted[1]
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}], d.formatted[1]
end

test 'time formatted with specified timezone, using area name' do
Expand All @@ -344,7 +349,7 @@ def create_driver(conf = CONFIG, opts = {})
d.feed(time, {"a"=>1})
end
assert_equal 1, d.formatted.size
assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n], d.formatted[0]
assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
end

test 'time formatted with specified timezone, using offset' do
Expand All @@ -358,7 +363,7 @@ def create_driver(conf = CONFIG, opts = {})
d.feed(time, {"a"=>1})
end
assert_equal 1, d.formatted.size
assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}\n], d.formatted[0]
assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
end

test 'configuration error raised for invalid timezone' do
Expand Down Expand Up @@ -402,7 +407,7 @@ def check_gzipped_result(path, expect)
end

assert File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}])
end
end

Expand Down Expand Up @@ -469,7 +474,7 @@ def parse_system(text)
end

path = d.instance.last_written_path
check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}\n])
check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}#{@default_newline}] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}#{@default_newline}])
end

test 'ltsv' do
Expand All @@ -482,7 +487,7 @@ def parse_system(text)
end

path = d.instance.last_written_path
check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z#{@default_newline}] + %[a:2\ttime:2011-01-02T13:14:15Z#{@default_newline}])
end

test 'single_value' do
Expand All @@ -495,13 +500,13 @@ def parse_system(text)
end

path = d.instance.last_written_path
check_gzipped_result(path, %[1\n] + %[2\n])
check_gzipped_result(path, %[1#{@default_newline}] + %[2#{@default_newline}])
end
end

test 'path with index number' do
time = event_time("2011-01-02 13:14:15 UTC")
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]

write_once = ->(){
d = create_driver
Expand Down Expand Up @@ -532,7 +537,7 @@ def parse_system(text)

test 'append' do
time = event_time("2011-01-02 13:14:15 UTC")
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]

write_once = ->(){
d = create_driver %[
Expand Down Expand Up @@ -567,7 +572,7 @@ def parse_system(text)
test 'append when JST' do
with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
time = event_time("2011-01-02 03:14:15+09:00")
formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}\n] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}\n]
formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}#{@default_newline}]

write_once = ->(){
d = create_driver %[
Expand Down Expand Up @@ -603,7 +608,7 @@ def parse_system(text)
test 'append when UTC-02 but timekey_zone is +0900' do
with_timezone("UTC-02") do # +0200
time = event_time("2011-01-02 17:14:15+02:00")
formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}\n] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}\n]
formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}#{@default_newline}]

write_once = ->(){
d = create_driver %[
Expand Down
9 changes: 7 additions & 2 deletions test/plugin_helper/test_compat_parameters.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ class CompatParameterTest < Test::Unit::TestCase
setup do
Fluent::Test.setup
@i = nil
@default_newline = if Fluent.windows?
"\r\n"
else
"\n"
end
end

teardown do
Expand Down Expand Up @@ -226,7 +231,7 @@ def write(chunk); end # dummy
t = event_time('2016-06-24 16:05:01') # localtime
iso8601str = Time.at(t.to_i).iso8601
formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
end

test 'plugin helper setups time injecting as unix time (integer from epoch)' do
Expand Down Expand Up @@ -260,7 +265,7 @@ def write(chunk); end # dummy
t = event_time('2016-06-24 16:05:01') # localtime
iso8601str = Time.at(t.to_i).iso8601
formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
end
end

Expand Down

0 comments on commit 73b4fd8

Please sign in to comment.