in_tail: Remove deleted paths in pos_file

Fix #3433 Signed-off-by: Takuro Ashie <ashie@clear-code.com>
fluent · Jul 25, 2021 · 86b2e4b · 86b2e4b
1 parent d2867c0
commit 86b2e4b
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 5 deletions.
diff --git a/lib/fluent/plugin/in_tail/position_file.rb b/lib/fluent/plugin/in_tail/position_file.rb
@@ -158,7 +158,7 @@ def fetch_compacted_entries(existing_targets = nil)
           end
         end
 
-        entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
+        entries = remove_deleted_files_entries(entries, existing_targets)
         entries
       end
 

diff --git a/test/plugin/in_tail/test_position_file.rb b/test/plugin/in_tail/test_position_file.rb
@@ -22,6 +22,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     invalidpath100000000000000000000000000000000
     unwatched\t#{UNWATCHED_STR}\t0000000000000000
   EOF
+  TEST_CONTENT_PATHS = {
+    "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -36,7 +40,11 @@ def follow_inodes_block
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+    paths = {
+      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
+    }
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -118,7 +126,7 @@ def follow_inodes_block
 
     test 'should ignore initial existing files on follow_inode' do
       write_data(@file, TEST_CONTENT)
-      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, {}, **{logger: $log})
+      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})
       @file.seek(0)
       assert_equal([], @file.readlines)
 
@@ -138,7 +146,12 @@ def follow_inodes_block
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      invalid_path = "invalidpath100000000000000000000000000000000"
+      paths = TEST_CONTENT_PATHS.merge({
+        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
+        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
+      })
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -147,6 +160,15 @@ def follow_inodes_block
       assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
     end
 
+    test 'compact deleted paths' do
+      write_data(@file, TEST_CONTENT)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal [], lines
+    end
+
     test 'compact data if duplicated line' do
       write_data(@file, <<~EOF)
         valid_path\t0000000000000002\t0000000000000001
@@ -163,7 +185,7 @@ def follow_inodes_block
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]