Kafka와 NLog를 활용한 로그 수집 및 Kibana 시각화

NuGet 패키지 참조

<PackageReference Include="NLog.Kafka" Version="0.2.1" />
<PackageReference Include="NLog.Web.AspNetCore" Version="4.14.0" />

Logstash 구성

input {
    kafka {
       bootstrap_servers => "127.0.0.1:9092"
       group_id => "logstash"
       topics => "logs"
       codec => "json"
    }
}
output{
  elasticsearch {
        hosts => ["127.0.0.1:9002"]
        index => "app_logs_%{+YYYY.MM.dd}"
  }
}

프로젝트 NLog 구성 (nlog.config)

<?xml version="1.0" encoding="utf-8" ?>
<nlog xmlns="http://www.nlog-project.org/schemas/NLog.xsd"
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
      autoReload="true"
      throwConfigExceptions="true"
      internalLogLevel="info"
      internalLogFile="App_Data/logs/internal-nlog.txt">

    <extensions>
        <add assembly="NLog.Web.AspNetCore"/>
        <add assembly="NLog.Kafka"/>
    </extensions>

    <variable name="consoleLayout" value="${longdate} | ${level} | ${logger} | ${message}" />
    <variable name="fileLayout" value="${longdate} | ${level} | ${logger} | ${message} | url: ${aspnet-request-url}" />

    <targets>
        <target name="console" xsi:type="ColoredConsole" layout="${consoleLayout}" />
        <target name="logFile" xsi:type="File" fileName="C:/logs/${shortdate}/logfile.log" archiveEvery="Day" />
        <target name="kafkaTarget" xsi:type="Kafka" topic="applicationLogs" bootstrapServers="localhost:9092">
            <layout xsi:type="JsonLayout">
                <attribute name="timestamp" layout="${longdate}" />
                <attribute name="level" layout="${level}" />
                <attribute name="logger" layout="${logger}" />
                <attribute name="message" layout="${message}" />
                <attribute name="exception" layout="${exception:tostring}" />
            </layout>
        </target>
    </targets>

    <rules>
        <logger name="Microsoft.*" maxLevel="Warn" final="true" />
        <logger name="*" minLevel="Debug" writeTo="console, logFile, kafkaTarget" />
    </rules>
</nlog>

코드 예제

public class LogHelper
{
    private static bool _isKafkaEnabled = false;
    private static bool _isFileEnabled = false;

    public const string PROPERTY_LOG_NAME = "logName";
    public const string PROPERTY_HOST_NAME = "hostName";

    public const string TARGET_CONSOLE = "console";
    public const string TARGET_FILE = "logFile";
    public const string TARGET_KAFKA = "kafkaTarget";

    public static Logger ConsoleLogger { get; private set; }
    public static Logger FileLogger { get; private set; }
    public static Logger KafkaLogger { get; private set; }

    static LogHelper()
    {
        Initialize();
    }

    private static void Initialize()
    {
        ConsoleLogger = LogManager.GetLogger(TARGET_CONSOLE);
        FileLogger = LogManager.GetLogger(TARGET_FILE);
        KafkaLogger = LogManager.GetLogger(TARGET_KAFKA);

        CheckTargets();
    }

    private static void CheckTargets()
    {
        var kafkaTarget = (NLog.Targets.Kafka.KafkaTarget)KafkaLogger.Factory.Configuration.FindTargetByName(TARGET_KAFKA);
        if (kafkaTarget != null && !string.IsNullOrEmpty(kafkaTarget.BootstrapServers))
        {
            _isKafkaEnabled = true;
        }

        var fileTarget = (NLog.Targets.FileTarget)FileLogger.Factory.Configuration.FindTargetByName(TARGET_FILE);
        if (fileTarget != null && fileTarget.FileName != null)
        {
            _isFileEnabled = true;
        }
    }

    public static void LogInfo(string message, string folderName = "")
    {
        Write(LogLevel.Info, message, folderName);
    }

    public static void LogWarning(string message, string folderName = "")
    {
        Write(LogLevel.Warn, message, folderName);
    }

    public static void LogError(Exception exception, string message = "", string folderName = "")
    {
        Write(LogLevel.Error, message, folderName, exception);
    }

    private static void Write(LogLevel level, string message, string folderName = "", Exception exception = null)
    {
        var logger = GetLoggerForTarget();

        var logEvent = new LogEventInfo(level, logger.Name, message);
        logEvent.Properties[PROPERTY_LOG_NAME] = folderName;
        logEvent.Properties[PROPERTY_HOST_NAME] = Environment.MachineName;
        logEvent.Exception = exception;

        logger.Log(logEvent);
    }

    private static Logger GetLoggerForTarget()
    {
        return _isKafkaEnabled ? KafkaLogger : _isFileEnabled ? FileLogger : ConsoleLogger;
    }
}

사용자 정의 레이아웃 렌더러

[LayoutRenderer(LogHelper.PROPERTY_LOG_NAME)]
[ThreadSafe]
public class CustomLogNameRenderer : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (logEvent.Properties.TryGetValue(LogHelper.PROPERTY_LOG_NAME, out var value) && value is string logName)
        {
            builder.Append(logName);
        }
    }
}

[LayoutRenderer(LogHelper.PROPERTY_HOST_NAME)]
[ThreadSafe]
public class CustomHostNameRenderer : LayoutRenderer
{
    protected override void Append(StringBuilder builder, LogEventInfo logEvent)
    {
        if (logEvent.Properties.TryGetValue(LogHelper.PROPERTY_HOST_NAME, out var value) && value is string hostName)
        {
            builder.Append(hostName);
        }
    }
}

Kibana 설정

Kibana는 Elasticsearch에서 수집된 데이터를 시각적으로 표현하는 도구입니다. 위의 Logstash 구성에서 생성된 인덱스 패턴을 사용하여 대시보드를 구성할 수 있습니다.