干货 | 基于 Elastic Stack 的日志分析系统实践（二）！

　　Elastic Stack也被称为ELK Stack，其能够安全可靠地获取任何来源、任何格式的数据，然后实时地对数据进行搜索、分析和可视化。上一篇文章为大家简单的介绍了一下Elastic Stack以及ElasticSearch 和 Lucene 的关系等内容，接下来为大家介绍一下ElasticSearch 单点部署和ElasticStack 分布式集群部署等内容，希望对你们有用。

　　推荐阅读：干货 | 基于Elastic Stack的日志分析系统实践

　　ElasticSearch单点部署

　　下载指定的ES版本

# 详细步骤见视频。

# 参考链接:https://www.elastic.co/cn/downloads/elasticsearch

　　部署JDK环境-可选步骤

　　单点部署elasticsearch

# 安装服务
yum -y localinstal elasticsearch-7.17.3-x86_64.rpm

# 修改配置文件
egrep -v "^#|^$" /etc/elasticsearch/elasticsearch.yml 
cluster.name: oldboyedu-elk
node.name: oldboyedu-elk103
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.0.0.103
discovery.seed_hosts: ["10.0.0.103"]

# 启动服务
systemctl start elasticsearch.service

　　OpenJDK切换Oracle JDK并修改堆内存大小

# 修改es的环境变量配置文件
vim /etc/sysconfig/elasticsearch
...
ES_JAVA_HOME=/oldboyedu/softwares/jdk


# 修改堆内存大小
vim /etc/elasticsearch/jvm.options
...
-Xms256m
-Xmx256m


# 验证堆内存大小
jmap -heap `ps -ef | grep java | grep -v grep | awk '{print $2}'`


# 同步配置文件到其他节点
data_rsync.sh /etc/sysconfig/elasticsearch
data_rsync.sh /etc/elasticsearch/jvm.options

　　ElasticStack分布式集群部署

　　elk101修改配置文件

egrep -v "^$|^#"  /etc/elasticsearch/elasticsearch.yml 
...
cluster.name: oldboyedu-elk
node.name: elk101
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
discovery.seed_hosts: ["elk101","elk102","elk103"]
cluster.initial_master_nodes: ["elk101","elk102","elk103"]

# 温馨提示:"node.name"各个节点配置要区分清楚，建议写对应的主机名称。

　　同步配置文件到集群的其他节点

# elk101同步配置文件到集群的其他节点
data_rsync.sh /etc/elasticsearch/elasticsearch.yml


# elk102节点配置
vim /etc/elasticsearch/elasticsearch.yml 
...
node.name: elk102


# elk103节点配置
vim /etc/elasticsearch/elasticsearch.yml 
...
node.name: elk103

　　所有节点删除之前的临时数据

pkill java
rm -rf /var/{lib,log}/elasticsearch/* /tmp/*
ll /var/{lib,log}/elasticsearch/ /tmp/

　　所有节点启动服务

# 所有节点启动服务
systemctl start elasticsearch

# 启动过程中建议查看日志
tail -100f /var/log/elasticsearch/oldboyedu-elk.log 

　　验证集群是否正常

curl elk103:9200/_cat/nodes?v

　　部署kibana服务

　　本地安装kibana

yum -y localinstall kibana-7.17.3-x86_64.rpm 

　　修改kibana的配置文件

vim /etc/kibana/kibana.yml
...
server.host: "10.0.0.101"
server.name: "oldboyedu-kibana-server"
elasticsearch.hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]
i18n.locale: "zh-CN"

　　启动kibana服务

systemctl enable --now kibana
systemctl status kibana

　　filebeat部署及基础使用

　　部署filebeat环境

yum -y localinstall filebeat-7.17.3-x86_64.rpm 

# 温馨提示:elk102节点操作。

　　修改filebeat的配置文件

# 编写测试的配置文件
mkdir /etc/filebeat/config
cat > /etc/filebeat/config/01-stdin-to-console.yml <<'EOF'
# 指定输入的类型
filebeat.inputs:
# 指定输入的类型为"stdin",表示标准输入
- type: stdin

# 指定输出的类型
output.console:
  # 打印漂亮的格式
  pretty: true
EOF

# 运行filebeat实例
filebeat -e -c /etc/filebeat/config/01-stdin-to-console.yml 

# 测试#见视频。

　　input的log类型

filebeat.inputs:
- type: log
  paths:
    - /tmp/test.log

output.console:
  pretty: true

　　input的通配符案例

filebeat.inputs:
- type: log
  paths:
    - /tmp/test.log
    - /tmp/*.txt

output.console:
  pretty: true

　　input的通用字段案例

filebeat.inputs:
- type: log
  # 是否启动当前的输入类型，默认值为true
  enabled: true
  # 指定数据路径
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  # 给当前的输入类型搭上标签
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]  
  # 自定义字段
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy" 
    hobby: "linux,抖音" 
  # 将自定义字段的key-value放到顶级字段.
  # 默认值为false，会将数据放在一个叫"fields"字段的下面.
  fields_under_root: true

output.console:
  pretty: true

　　将数据写入es案例

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]  
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy" 
    hobby: "linux,抖音" 
  fields_under_root: true


output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]

　　自定义es索引名称

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]  
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy" 
    hobby: "linux,抖音" 
  fields_under_root: true


output.elasticsearch:
  enabled: true
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 
  index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式 
setup.template.pattern: "oldboyedu-linux*"

　　多个索引写入案例

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]  
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy" 
    hobby: "linux,抖音" 
  fields_under_root: true


output.elasticsearch:
  enabled: true
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 
  # index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}"
  indices:
    - index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}"
      # 匹配指定字段包含的内容
      when.contains:
        tags: "oldboyedu-linux80"
    - index: "oldboyedu-linux-python-%{+yyyy.MM.dd}"
      when.contains:
        tags: "oldboyedu-python"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式 
setup.template.pattern: "oldboyedu-linux*"

　　自定义分片和副本案例

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /tmp/test.log
    - /tmp/*.txt
  tags: ["oldboyedu-linux80","容器运维","DBA运维","SRE运维工程师"]  
  fields:
    school: "北京昌平区沙河镇"
    class: "linux80"

- type: log
  enabled: true
  paths:
    - /tmp/test/*/*.log
  tags: ["oldboyedu-python","云原生开发"]
  fields:
    name: "oldboy" 
    hobby: "linux,抖音" 
  fields_under_root: true


output.elasticsearch:
  enabled: true
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 
  # index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}"
  indices:
    - index: "oldboyedu-linux-elk-%{+yyyy.MM.dd}"
      # 匹配指定字段包含的内容
      when.contains:
        tags: "oldboyedu-linux80"
    - index: "oldboyedu-linux-python-%{+yyyy.MM.dd}"
      when.contains:
        tags: "oldboyedu-python"

# 禁用索引生命周期管理
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "oldboyedu-linux"
# 设置索引模板的匹配模式 
setup.template.pattern: "oldboyedu-linux*"
# 覆盖已有的索引模板
setup.template.overwrite: false
# 配置索引模板
setup.template.settings:
  # 设置分片数量
  index.number_of_shards: 3
  # 设置副本数量，要求小于集群的数量
  index.number_of_replicas: 2

　　为了让学员学以致用，老男孩教育不断更新Linux培训教学大纲，与一众讲师精心打造Linux云计算SRE课程体系。老男孩教育Linux云计算SRE运维课程，从零基础开始授课，适合零基础且对Linux感兴趣的学员学习。课程的主要目的就是培养出专业的Linux系统工程师，名师授课全程面授教学，边讲边练帮助学员高效掌握Linux技术。