Merge pull request #3352 from kenhys/rotate-in-system-log

Enable system.log.rotate_... system configuration

lib/fluent/command/fluentd.rb

@@ -85,9 +85,8 @@
   opts[:log_path] = s
 }
 
-ROTATE_AGE = %w(daily weekly monthly)
 op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
-  if ROTATE_AGE.include?(age)
+  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
     opts[:log_rotate_age] = age
   else
     begin

lib/fluent/log.rb

@@ -49,6 +49,7 @@ module TTYColor
     LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
 
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
+    LOG_ROTATE_AGE = %w(daily weekly monthly)
 
     def self.str_to_level(log_level_str)
       case log_level_str.downcase

lib/fluent/supervisor.rb

@@ -518,6 +518,8 @@ def init(process_type, worker_id)
         dl_opts = {}
         # subtract 1 to match serverengine daemon logger side logging severity.
         dl_opts[:log_level] = @level - 1
+        dl_opts[:log_rotate_age] = @log_rotate_age if @log_rotate_age
+        dl_opts[:log_rotate_size] = @log_rotate_size if @log_rotate_size
         logger = ServerEngine::DaemonLogger.new(@logdev, dl_opts)
         $log = Fluent::Log.new(logger, @opts)
         $log.enable_color(false) if @path
@@ -606,6 +608,19 @@ def initialize(opt)
 
       @cl_opt = opt
       @conf = nil
+      # parse configuration immediately to initialize logger in early stage
+      if @config_path and File.exist?(@config_path)
+        @conf = Fluent::Config.build(config_path: @config_path,
+                                     encoding: @conf_encoding ? @conf_encoding : 'utf-8',
+                                     additional_config: @inline_config ? @inline_config : nil,
+                                     use_v1_config: !!@use_v1_config)
+        @system_config = build_system_config(@conf)
+        if @system_config.log
+          @log_rotate_age ||= @system_config.log.rotate_age
+          @log_rotate_size ||= @system_config.log.rotate_size
+        end
+        @conf = nil
+      end
 
       log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
                   ignore_same_log_interval: opt[:ignore_same_log_interval]}

lib/fluent/system_config.rb

@@ -55,6 +55,20 @@ class SystemConfig
     config_section :log, required: false, init: true, multi: false do
       config_param :format, :enum, list: [:text, :json], default: :text
       config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S %z'
+      config_param :rotate_age, default: nil do |v|
+        if Fluent::Log::LOG_ROTATE_AGE.include?(v)
+          v.to_sym
+        else
+          begin
+            Integer(v)
+          rescue ArgumentError => e
+            raise Fluent::ConfigError, e.message
+          else
+            v.to_i
+          end
+        end
+      end
+      config_param :rotate_size, :size, default: nil
     end
 
     config_section :counter_server, multi: false do

test/config/test_system_config.rb

@@ -143,5 +143,51 @@ def parse_text(text)
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(level, sc.log_level)
     end
+
+    sub_test_case "log rotation" do
+      data('daily' => "daily",
+           'weekly' => 'weekly',
+           'monthly' => 'monthly')
+      test "symbols for rotate_age" do |age|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age #{age}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(age.to_sym, sc.log.rotate_age)
+      end
+
+      test "numeric number for rotate age" do
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+            </log>
+          </system>
+        EOS
+        s = FakeSupervisor.new
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(3, sc.log.rotate_age)
+      end
+
+      data(h: ['100', 100],
+           k: ['1k', 1024],
+           m: ['1m', 1024 * 1024],
+           g: ['1g', 1024 * 1024 * 1024])
+      test "numeric and SI prefix for rotate_size" do |(label, size)|
+        conf = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_size #{label}
+            </log>
+          </system>
+        EOS
+        sc = Fluent::SystemConfig.new(conf)
+        assert_equal(size, sc.log.rotate_size)
+      end
+    end
   end
 end

test/test_config.rb

@@ -160,7 +160,8 @@ def test_inline
     prepare_config
     opts = {
       :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
+      :use_v1_config => false
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)

test/test_supervisor.rb

@@ -7,6 +7,7 @@
 require 'net/http'
 require 'uri'
 require 'fileutils'
+require 'tempfile'
 
 if Fluent.windows?
   require 'win32/event'
@@ -489,6 +490,40 @@ def test_logger_with_rotate_age_and_rotate_size(rotate_age)
     assert_equal 10, $log.out.instance_variable_get(:@shift_size)
   end
 
+  sub_test_case "system log rotation" do
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_override_default_log_rotate
+      Tempfile.open do |file|
+        config = parse_text(<<-EOS)
+          <system>
+            <log>
+              rotate_age 3
+              rotate_size 300
+            </log>
+          </system>
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
+  end
+
   def test_inline_config
     omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'