实战ELK收集应用日志
|
0 1 2 3 4 5 6 7 8 9 |
$ uname -a Linux Elastics-01.t4x.org 3.10.0-862.el7.x86_64 #1 SMP Fri Apr 20 16:44:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux $ mkdir /byrd/tools -p $ cd /byrd/tools $ tar zxf jdk-8u191-linux-x64.tar.gz $ ln -s /opt/jdk1.8.0_191 /usr/local/jdk $ echo 'export PATH="$PATH:/usr/local/jdk/bin"' >> /etc/profile $ source /etc/profile $ java -version java version "1.8.0_191" |
Elasticsearch:安装
|
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.4.tar.gz $ tar zxf elasticsearch-6.2.4.tar.gz $ mv elasticsearch-6.2.4 /opt/ $ ln -s /opt/elasticsearch-6.2.4 /usr/local/elasticsearch $ groupadd app -g 2768 $ useradd app -u 2768 -g app -m -s /bin/bash $ chown -R app.app /usr/local/elasticsearch/ $ mkdir -p /data/elasticsearch/{logs,data} $ chown -R app.app /data/elasticsearch/{logs,data} $ sed -i 's#\#cluster.name: my-application#cluster.name: tomcat-app#g' /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#path.data: /path/to/data#path.data: /data/elasticsearch/data#g' /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#path.logs: /path/to/logs#path.logs: /data/elasticsearch/logs#g' /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#network.host: 192.168.0.1#network.host: 0.0.0.0#g' /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#http.port: 9200#http.port: 9200#g' /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#discovery.zen.ping.unicast.hosts: \["host1", "host2"\]#discovery.zen.ping.unicast.hosts: \["192.168.227.23", "192.168.227.24", "192.168.227.25"\]#g' /usr/local/elasticsearch/config/elasticsearch.yml $ echo "http.cors.enabled: true" >> /usr/local/elasticsearch/config/elasticsearch.yml $ echo 'http.cors.allow-origin: "*"' >> /usr/local/elasticsearch/config/elasticsearch.yml $ sed -i 's#\#node.name: node-1#node.name: node-1#g' /usr/local/elasticsearch/config/elasticsearch.yml #第一台 $ sed -i 's#\#node.name: node-1#node.name: node-2#g' /usr/local/elasticsearch/config/elasticsearch.yml #第二台 $ sed -i 's#\#node.name: node-1#node.name: node-3#g' /usr/local/elasticsearch/config/elasticsearch.yml #第三台 $ echo '* - nofile 65536' >>/etc/security/limits.conf $ echo '* - nproc 4096' >>/etc/security/limits.conf $ echo "vm.max_map_count=655360" >> /etc/sysctl.conf $ sysctl -p $ curl http://127.0.0.1:9200 { "name" : "node-2", "cluster_name" : "tomcat-app", "cluster_uuid" : "qOWI7ZebQRSRNamdf5WEHw", "version" : { "number" : "6.2.4", "build_hash" : "ccec39f", "build_date" : "2018-04-12T20:37:28.497551Z", "build_snapshot" : false, "lucene_version" : "7.2.1", "minimum_wire_compatibility_version" : "5.6.0", "minimum_index_compatibility_version" : "5.0.0" }, "tagline" : "You Know, for Search" } |
|
0 1 2 3 4 |
$ https://artifacts.elastic.co/downloads/logstash/logstash-6.2.4.tar.gz $ tar zxf logstash-6.2.4.tar.gz $ mv logstash-6.2.4 /opt/ $ ln -s /opt/logstash-6.2.4/ /usr/local/logstash $ mkdir /usr/local/logstash/conf |
|
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 |
input { redis { host => "10.1.20.106" port => 6379 password => "N2.heeo9Q" db => "8" data_type => "list" key => "logstash" } } filter { if [app] == "nginx" { if [type] == "nginx-access" { json { source => "message" remove_field => ["message"] } geoip { source => "remote_addr" target => "geoip" database => "/opt/GeoLite2-City.mmdb" add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}"] add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}"] } mutate { convert => ["[geoip][coordinates]", "float"] } } } if [app] == "tomcat" { grok { match => ["message", "%{TIMESTAMP_ISO8601:logdate}\ (%{LOGLEVEL:loglevel}|\[\ %{LOGLEVEL:loglevel}\]|\[%{LOGLEVEL:loglevel}\])(-|)\ ([A-Za-z0-9_. -]+|%{JAVACLASS:class})\ %{JAVACLASS:class}"] } mutate { remove_field => ["@version","prospector","beat","offset"] } date { locale => "en" match => ["logdate", "yyyy-MM-dd HH:mm:ss"] } } } output { elasticsearch { hosts => ["http://10.1.20.108:9200","http://10.1.20.107:9200","http://10.1.20.106:9200"] index => "logs-%{type}-%{+YYYY.MM.dd}" } } |
|
0 1 2 3 4 |
$ wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.2.4-linux-x86_64.tar.gz $ tar zxf filebeat-6.2.4-linux-x86_64.tar.gz $ ln -s /opt/filebeat-6.2.4-linux-x86_64/ /usr/local/filebeat $ mkdir /usr/local/filebeat/conf $ nohup /usr/local/filebeat/filebeat -e -c /usr/local/filebeat/conf/fields.yml >/data/logs/filebeat/filebeat.log & |
|
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
$ cat /usr/local/filebeat/conf/fields.yml filebeat.prospectors: - type: log paths: - /data/logs/nginx/jiayou_manage_json.log tags: ["local", "nginx"] fields: app: nginx type: nginx-access fields_under_root: true - type: log paths: - /data/logs/tomcat/tomcat.log tags: ["local", "tomcat-logs-01"] fields: app: tomcat type: local-tomcat-personal fields_under_root: true multiline: pattern: '^\d+-\d+-\d+ \d+:\d+:\d+' negate: true match: after - type: log paths: - /data/logs/tomcat/catalina.out tags: ["local", "tomcat-logs-02"] fields: app: tomcat type: local-tomcat-default fields_under_root: true multiline: pattern: '^\d+-\d+-\d+ \d+:\d+:\d+' negate: true match: after output.redis: hosts: ["192.168.227.24"] password: "admin" key: "logstash" db: 8 datatype: list |
|
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
# https://github.com/mobz/elasticsearch-head $ git clone git://github.com/mobz/elasticsearch-head.git $ wget https://nodejs.org/dist/v10.16.0/node-v10.16.0-linux-x64.tar.xz $ xz -d node-v10.16.0-linux-x64.tar.xz $ tar xf node-v10.16.0-linux-x64.tar $ mv node-v10.16.0-linux-x64 /opt/ $ ln -s /opt/node-v10.16.0-linux-x64/ /usr/local/node $ ln -s /usr/local/node/bin/node /usr/bin/node $ cd elasticsearch-head $ yum install bzip2 $ /usr/local/node/bin/npm install -g grunt $ /usr/local/node/bin/npm install -g grunt-cli $ /usr/local/node/bin/npm install phantomjs --unsafe-perm $ /usr/local/node/bin/npm install --unsafe-perm $ /usr/local/node/bin/npm run start |
|
0 1 2 3 4 5 6 |
$ wget https://artifacts.elastic.co/downloads/kibana/kibana-6.2.4-linux-x86_64.tar.gz $ mv kibana-6.2.4-linux-x86_64 /opt/ $ ln -s /opt/kibana-6.2.4-linux-x86_64 /usr/local/kibana sed -i 's/#server.port: 5601/server.port: 5601/g' /usr/local/kibana/config/kibana.yml sed -i 's/#server.host: "localhost"/server.host: "192.168.227.24"/g' /usr/local/kibana/config/kibana.yml sed -i 's/\#elasticsearch.url: \"http:\/\/localhost:9200\"/elasticsearch.url: \"http:\/\/192.168.227.24:9200\"/g' /usr/local/kibana/config/kibana.yml $ nohup /usr/local/kibana/bin/kibana >/data/logs/kibana/kibana.log & |
参考文档
1:https://segmentfault.com/a/1190000013317848文 章 源 自 note.t4x.orgByrd's Blog-https://note.t4x.org/project/elk-collect-log/
文 章 源 自 note.t4x.orgByrd's Blog-https://note.t4x.org/project/elk-collect-log/
申明：除非注明Byrd's Blog内容均为原创,未经许可禁止转载!详情请阅读版权申明!
