1. 구축 환경 및 소프트웨어 버전

2. JVM 설치 (모든 서버)

tar xf jdk-11.0.7_linux-x64_bin.tar.gz -C /usr/local/
vim /etc/profile.d/java.sh
# 기존 JVM 환경변수가 있으면 삭제 필요

export JAVA_HOME=/usr/local/jdk-11.0.7/
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar 

source /etc/profile.d/java.sh
java -version
java version "11.0.7" 2020-04-14 LTS
Java(TM) SE Runtime Environment 18.9 (build 11.0.7+8-LTS)
Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.7+8-LTS, mixed mode)

3. Elasticsearch 클러스터 구축

3.1 ES 설치 및 설정

tar xf elasticsearch-7.6.2-linux-x86_64.tar.gz -C /usr/local/
cd /usr/local/
ln -sv elasticsearch-7.6.2/ elasticsearch
cd elasticsearch/config/
grep "^[a-Z]" /usr/local/elasticsearch/config/elasticsearch.yml

cluster.name: app-elk-cluster
node.name: node-1
path.data: /Data/es/data
path.logs: /Data/es/log
bootstrap.memory_lock: true
network.host: 192.168.1.101
http.port: 9200
discovery.seed_hosts: ["192.168.1.101", "192.168.1.102", "192.168.1.103"]
cluster.initial_master_nodes: ["192.168.1.101", "192.168.1.102", "192.168.1.103"]
discovery.zen.minimum_master_nodes: 2
gateway.recover_after_nodes: 2

다른 노드 설정값:

network.host:   # 서버 IP 주소
node.name:   # 노드 고유 이름

3.2 사용자 생성 및 디렉토리 설정

mkdir -pv /Data/es/
useradd elasticsearch
chown -R elasticsearch:elasticsearch /Data/es/
chown -R elasticsearch:elasticsearch /usr/local/elasticsearch-7.6.2/

3.3 시스템 파라미터 설정

tail /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096
* soft memlock unlimited
* hard memlock unlimited
echo "vm.max_map_count=262144" >> /etc/sysctl.conf
sysctl -p
reboot

3.4 서비스 시작 (3대 동시 시작)

su - elasticsearch
cd /usr/local/elasticsearch
nohup ./bin/elasticsearch > /tmp/es.log &
tailf /tmp/es.log

로그에 다음 내용이 표시되면 정상:

master node changed {previous [], current [{node-1}{TB8YdpzNR7xH2ZJq8gO-P}{GPgUcAoOSgTiLoIfrktc-B}{192.168.1.101}{192.168.1.101:9300}

3.5 클러스터 상태 확인

netstat -tnlp|grep -E "9200|9300"
curl http://192.168.1.103:9200/
{
  "name" : "node-3",
  "cluster_name" : "app-elk-cluster",
  "cluster_uuid" : "mSE1bV1UTh-p1VSPLLQLLQ",
  "version" : {
    "number" : "7.6.2",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "ef48eb35cf30adf4db14086e8aabd07ef6fb113f",
    "build_date" : "2020-03-26T06:34:37.794943Z",
    "build_snapshot" : false,
    "lucene_version" : "8.4.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

4. Kibana 설치

4.1 Kibana 설치 및 설정

tar xf kibana-7.6.2-linux-x86_64.tar.gz -C /usr/local/
cd /usr/local/
ln -sv kibana-7.6.2-linux-x86_64/ kibana
cd kibana/config
grep "^[a-Z]" /usr/local/kibana/config/kibana.yml

server.port: 5601
server.host: "192.168.1.103"
elasticsearch.hosts: ["http://192.168.1.101:9200"]
i18n.locale: "zh-CN"

4.2 서비스 시작

nohup ./kibana --allow-root > /tmp/kibana.log &
tailf /tmp/kibana.log
# 다음과 같은 메시지 확인
"tags":["listening","info"],"pid":13922,"message":"Server running at http://192.168.1.103:5601"}

4.3 Kibana 접근

웹 브라우저에서 http://192.168.1.103:5601 접속

5. Zookeeper/Kafka 클러스터 설치

5.1 Zookeeper 설치 및 설정

tar xf kafka_2.12-2.3.1.tgz -C /usr/local/
cd /usr/local/
ln -sv kafka_2.12-2.3.1 kafka
cd kafka/config/
grep "^[a-Z]" /usr/local/kafka/config/zookeeper.properties

dataDir=/Data/zookeeper
clientPort=2181
maxClientCnxns=0
tickTime=2000
initLimit=20
syncLimit=10
server.1=192.168.1.111:2888:3888
server.2=192.168.1.112:2888:3888
server.3=192.168.1.113:2888:3888

mkdir -pv /Data/zookeeper
echo "1" > /Data/zookeeper/myid

5.2 Kafka 설치 및 설정

grep "^[a-Z]" /usr/local/kafka/config/server.properties

broker.id=1
listeners=PLAINTEXT://192.168.1.111:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/Data/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.1.111:2181,192.168.1.112:2181,192.168.1.113:2181
zookeeper.connection.timeout.ms=20000
group.initial.rebalance.delay.ms=0

다른 노드 설정:

# Zookeeper
echo "x" > /Data/zookeeper/myid

# Kafka
broker.id=2  # 고유값
host.name=본인IP

5.3 Zookeeper 시작

nohup /usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties &
netstat -nlpt | grep -E "2181|2888|3888"

5.4 Kafka 시작

vim /etc/hosts
# 127.0.0.1 에 hostname 추가

/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties &

5.5 테스트

# Topic 생성
/usr/local/kafka/bin/kafka-topics.sh --create --zookeeper 192.168.1.111:2181 --replication-factor 2 --partitions 1 --topic web-log

# Topic 목록 확인
/usr/local/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.111:2181

# Topic 상세 정보
/usr/local/kafka/bin/kafka-topics.sh --describe --zookeeper 192.168.1.111:2181 --topic web-log

# 프로듀서 (메시지 전송)
/usr/local/kafka/bin/kafka-console-producer.sh --broker-list 192.168.1.111:9092 --topic web-log

# 컨슈머 (메시지 수신)
/usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.1.112:9092 --topic web-log --from-beginning

6. Filebeat 설치 및 설정

6.1 Filebeat 설치

tar xf filebeat-7.6.2-linux-x86_64.tar.gz -C /usr/local/
cd /usr/local/filebeat-7.6.2-linux-x86_64/
vim filebeat.yml

filebeat.inputs:

- type: log
  enabled: true
  json.keys_under_root: true
  json.overwrite_keys: true
  fields_under_root: true
  paths:
    - /opt/logs/nginx/access.log
  fields:
    type: log
    log_topic: app-nginx-access

output.kafka:
  enabled: true
  hosts: ["192.168.1.111:9092", "192.168.1.112:9092", "192.168.1.113:9092"]
  topic: '%{[log_topic]}'

6.2 Nginx 설치 및 JSON 로그 설정

# Nginx 설치 (생략)

vim nginx.conf
log_format json '{"@timestamp":"$time_iso8601",'
  '"@version":"1",'
  '"client_ip":"$remote_addr",'
  '"status":"$status",'
  '"host":"$server_addr",'
  '"url":"$request_uri",'
  '"domain":"$host",'
  '"size":"$body_bytes_sent",'
  '"responsetime":"$request_time",'
  '"referer":"$scheme://$server_addr$request_uri",'
  '"user_agent":"$http_user_agent"'
'}';
access_log /opt/logs/nginx/access.log json;

nginx -t -c /usr/local/nginx/conf/nginx.conf
nginx -c /usr/local/nginx/conf/nginx.conf

6.3 Filebeat 시작

nohup 실행:

nohup /usr/local/filebeat/filebeat -e -c /usr/local/filebeat/filebeat.yml > /tmp/filebeat.log &

systemd로 실행:

vim /etc/systemd/system/filebeat.service
[Unit]
Description=filebeat server daemon
Documentation=/usr/local/filebeat/filebeat -help
Wants=network-online.target
After=network-online.target

[Service]
User=root
Group=root
ExecStart=/usr/local/filebeat/filebeat -c /usr/local/filebeat/filebeat.yml --path.logs /usr/local/filebeat/logs
Restart=always

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl restart filebeat.service

Kafka Topic 확인:

/usr/local/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.111:2181

7. Logstash 설치 및 설정

tar xf logstash-7.6.2.tar.gz -C /usr/local/
cd /usr/local/logstash-7.6.2/config/
vim pipeline.conf

input {
    kafka {
        bootstrap_servers => "192.168.1.111:9092,192.168.1.112:9092,192.168.1.113:9092"
        topics => "app-nginx-access"
        codec => "json"
        consumer_threads => 5
        decorate_events => true
    }
}

output {
    elasticsearch {
        hosts => ["192.168.1.101:9200", "192.168.1.102:9200"]
        index => "app-nginx-access-%{+YYYY-MM-dd}"
    }
}

# 설정 검증
../bin/logstash -f pipeline.conf -t --verbose

# 백그라운드 실행
nohup /usr/local/logstash-7.6.2/bin/logstash -f pipeline.conf > /tmp/logstash.log &

검증: Kibana 웹 인터페이스에서 app-nginx-access-* 인덱스 존재 여부 확인