##################################################OS:CentOS Linux release 7.2.1511 (Core) #elk5.1.2运行对内核版本有要求#ELK-Version:elasticsearch-5.1.2kibana-5.1.2logstash-5.1.2filebeat-5.1.2#Base and JDK and ELKgroupadd elkuseradd -g elk elk passwd elkyum install vim lsof man wget ntpdate vixie-cron unzip supervisor -ycrontab -e*/1 * * * * /usr/sbin/ntpdate time.windows.com > /dev/null 2>&1systemctl restart crond.service ulimit -SHn 65536#禁用selinux，关闭iptablessed -i "s#SELINUX=enforcing#SELINUX=disabled#" /etc/selinux/configsystemctl status firewalld.serviceyum install iptables-services systemctl status iptables.servicereboottar -zxvf jdk-8u112-linux-x64.tar.gz -C /usr/local/vim /etc/profileexport JAVA_HOME=/usr/local/jdk1.8.0_112export JRE_HOME=/usr/local/jdk1.8.0_112/jreexport PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATHexport CLASSPATH=$CLASSPATH:.:$JAVA_HOME/lib:$JAVA_HOME/jre/libsource /etc/profile#Elasticsearch5.1.2#(cluster时在其他server安装elasticsearch，并配置相同集群名称，不同节点名称即可)#wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz#wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.deb#wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.rpmwget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.zipunzip elasticsearch-5.1.2.zip -d /usr/local/cd /usr/local/elasticsearch-5.1.2mkdir -p /data/{db,logs}chown elk:elk /data/{db,logs} -Rchown elk:elk elasticsearch-5.1.2 -Rvim config/elasticsearch.yml#cluster.name: elasticsearch #node.name: "es-node1" #node.master: true #node.data: true path.data: /data/dbpath.logs: /data/logsnetwork.host: 192.168.1.30http.port:9200discovery.zen.ping.unicast.hosts: ["192.168.1.30"]#http.cors.enabled: true#http.cors.allow-origin: "*"su - elkcd /usr/local/elasticsearch-5.1.2nohup ./bin/elasticsearch &#或su -c "/usr/local/elasticsearch-5.1.2/elasticsearch" elk#插件安装#http://blog.csdn.net/napoay/article/details/53896348#在elasticsearch5.x下Run elasticsearch-head as a standalone server#https://github.com/mobz/elasticsearch-head#下载 elasticsearch-head 或者git clone到随便一个文件夹#安装nodejs#cd /path/to/elasticsearch-head#npm install -g grunt-cli#npm install#grunt server#http://localhost:9100/#kibana-5.1.2#wget https://artifacts.elastic.co/downloads/kibana/kibana-5.1.2-x86_64.rpm#wget https://artifacts.elastic.co/downloads/kibana/kibana-5.1.2-amd64.debwget https://artifacts.elastic.co/downloads/kibana/kibana-5.1.2-linux-x86_64.tar.gztar -zxvf kibana-5.1.2-linux-x86_64.tar.gz -C /usr/local/cd /usr/local/kibana-5.1.2-linux-x86_64vim config/kibana.yml server.port: 5601server.host: "192.168.1.30"elasticsearch_url: "http://192.168.1.30:9200"cd /usr/local/kibana-5.1.2-linux-x86_64nohup ./bin/kibana -c config/kibana.yml #logstash-5.1.2#https://artifacts.elastic.co/downloads/logstash/logstash-5.1.2.tar.gz#https://artifacts.elastic.co/downloads/logstash/logstash-5.1.2.deb#https://artifacts.elastic.co/downloads/logstash/logstash-5.1.2.rpmwget https://artifacts.elastic.co/downloads/logstash/logstash-5.1.2.zipunzip logstash-5.1.2.zip -d /usr/local/cd /usr/local/logstash-5.1.2vim config/logstash.confinput { beats { host => "192.168.1.30" port => 5044 ssl => false ssl_certificate => "/etc/pki/tls/certs/logstash.crt" ssl_key => "/etc/pki/tls/private/logstash.key" ssl_certificate_authorities => ["/etc/pki/tls/certs/filebeat.crt"] ssl_verify_mode => "force_peer" }}filter { if [type] == "syslog" { grok { match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{host}" ] } syslog_pri { } date { match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] } }}output { elasticsearch { hosts => ["192.168.1.30:9200"] sniffing => false manage_template => false index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" document_type => "%{[@metadata][type]}" }}#这里通过grok对字符串文本信息进行处理#ssl：true 表示开启 Logstash 的 SSL/TLS 安全通信功能；#ssl_certificate_authorities：配置 Logstash 使其信任所有由该 CA证书发行的证书；#ssl_certificate & ssl_key：Logstash server 证书和 key 文件。Logstash使用它们向Filebeat端证明自己的可信身份；#ssl_verify_mode：表明Logstash是否验证Filebeat端证书。有效值有peer或force_peer。如果是force_peer，表示如果Filebeat没有提供证书，Logstash就会关闭连接生成Logstash自签名证书和key文件(ELK端执行)#两种方式：一种指定IP地址，一种指定fqdn(dns),选其一即可#1、IP地址在[ v3_ca ]配置段下设置上面的参数。192.168.1.30是logstash服务端的地址。vi /etc/pki/tls/openssl.cnfsubjectAltName = IP: 192.168.1.30cd /etc/pki/tlsopenssl req -config /etc/pki/tls/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash.key -out certs/logstash.crt#注意将-days设置大点，以免证书过期。#2、fqdn# 不需要修改openssl.cnf文件。cd /etc/pki/tlsopenssl req -subj '/CN=hostnameorip/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash.key -out certs/logstash.crt验证配置cd /usr/local/logstash-5.1.2./bin/logstash -t -f /usr/local/logstash-5.1.2/config/logstash.conf #Sending Logstash's logs to /usr/local/logstash-5.1.2/logs which is now configured via log4j2.properties#Configuration OK#[2017-01-20T14:25:05,731][INFO ][logstash.runner ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash启动logstashcd /usr/local/logstash-5.1.2nohup ./bin/logstash -f config/logstash.conf &#Nginxyum install nginx -yvim /etc/nginx/conf.d/elk.confserver { server_name elk.sudo.com; auth_basic "Restricted Access"; auth_basic_user_file passwd; location / { proxy_pass http://192.168.1.30:5601; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_set_header Host $host; proxy_cache_bypass $http_upgrade; }}#htpsswd添加--yum install httpd-tools –yecho -n 'sudo:' >> /etc/nginx/passwd #添加用户openssl passwd elk.sudo.com >> /etc/nginx/passwd #添加密码cat /etc/nginx/passwd #查看chkconfig nginx on && service nginx start #Filebeat-Logstash-Config#wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.1.2-x86_64.rpm#wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.1.2-amd64.deb#wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.1.2-linux-x86_64.tar.gzwget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.1.2-linux-x86_64.tar.gztar -zxvf filebeat-5.1.2-linux-x86_64.tar.gz -C /usr/local/#Filebeat与Logstash的通信用户可以使用TLS双向认证加密Filebeat和Logstash的连接，保证Filebeat只向可信的Logstash发送加密的数据。同样的，Logstash也只接收可信的Filebeat发送的数据生成Filebeat自签名证书和key文件(被监控端执行)，方法同上，对应的证书名称及key文件名称改为filebeat.*，对应的IP或主机名等修改#Logstash配置同Filebeat配置类似，把Filebeat端上的证书复制到Logstash上/etc/pki/tls/certs/路径，配置修改见logstash红色部分把Logstash的自签名证书传送到Filebeat所在服务器。对filebeat.yml的logstash output部分的tls配置作如下修改（logstash及filebeat端配置了ssl/tls，但并未启用）：tls: ## logstash的自签名证书。 certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key"其中：certificate_authorities：CA证书，即用来签署证书的证书。这里表示配置Filebeat使其信任所有由该CA证书发行的证书。因为自签名证书的发行者和证书主体相同，所以这里直接使用Logstash证书使Filebeat信任使用该证书的Logstash server；certificate & certificate_key：Filebeat证书和key文件。Filebeat将使用它们向Logstash server证明自己的可信身份(被监控端生成拷贝至本机)备份filebeat.yml，新建cd /usr/local/filebeat-5.1.2-linux-x86_64cp -a filebeat.full.yml filebeat.ymlvim filebeat.ymlfilebeat.prospectors:- input_type: log paths: - /var/log/messages document_type: syslogoutput.logstash: hosts: ["192.168.1.30:5044"]#output.elasticsearch: #hosts: ["localhost:9200"] #不直接发送到elasticsearch,注释这两行tls: ## logstash的自签名证书。 certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key"验证配置cd /usr/local/filebeat-5.1.2-linux-x86_64/filebeat -c filebeat.yml -configtestConfig OK启动filebeatnohup ./filebeat -c filebeat.yml首次访问elk.sudo.com，Index name or pattern输入filebeat-*，Time-field name选择@timestampELK的使用从此开始！#采集nginx日志(json化)进行监控：修改nginx日志格式log_format json '{"@timestamp":"$time_iso8601",''"slbip":"$remote_addr",''"clientip":"$http_x_forwarded_for",''"serverip":"$server_addr",''"size":"$body_bytes_sent",''"responsetime":"$request_time",''"domain":"$host",''"method":"$request_method",''"requesturi":"$request_uri",''"url":"$uri",''"appversion":"$HTTP_APP_VERSION",''"referer":"$http_referer",''"agent":"$http_user_agent",''"status":"$status",''"devicecode":"$HTTP_HA"}';access_log /var/log/nginx/access.log json;#json格式化工具：https://c.runoob.com/front-end/53#在线工具-oschina：http://tool.oschina.net/codeformat/json#菜鸟工具：https://c.runoob.com/#添加filebeat配置vim filebeat.ymlfilebeat.prospectors:- input_type: log paths: - /var/log/nginx/access.log document_type: nginxlog json.message_key:#output.elasticsearch: #hosts: ["192.168.1.30:9200"] #不直接发送到elasticsearch，因nginx日志格式已经json化，可以直接输出到elasticsearchoutput.logstash: hosts: ["192.168.1.30:5044"]tls: certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key" bulk_max_size: 1024logging.to_files: truelogging.files:#编写Logstash配置文件input { beats { host => "192.168.1.30" port => 5044 ssl => false ssl_certificate => "/etc/pki/tls/certs/logstash.crt" ssl_key => "/etc/pki/tls/private/logstash.key" ssl_certificate_authorities => ["/etc/pki/tls/certs/filebeat.crt"] ssl_verify_mode => "force_peer" }}output { if [type] == "nginxlog" { elasticsearch { hosts => ["192.168.1.30:9200"] index => "filebeat-nginxlog-%{+YYYY.MM.dd}" manage_template => true flush_size => 50000 idle_flush_time => 10}}}访问elk.sudo.com，Index name or pattern输入logstash-nginxlog-*，Time-field name选择json@timestamp，，创建添加#对nginx日志(其他可格式化日志)格式进行json化，通过filebeat发送至elasticsearch存储，对非json格式文件messages/secure等不能直接解析#采集nginx日志(grok）进行监控：nginx日志格式不变log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"';access_log /var/log/nginx/access.log main;#Grok Debug--Grok匹配工具：http://grokdebug.herokuapp.com/#在线工具-oschina：http://tool.oschina.net/codeformat/json#菜鸟工具：https://c.runoob.com/#添加filebeat配置vim filebeat.ymlfilebeat.prospectors:- input_type: log paths: - /var/log/nginx/access.log document_type: nginxlog #json.message_key:#output.elasticsearch: #hosts: ["192.168.1.30:9200"] #不直接发送到elasticsearch，因nginx日志格式已经json化，可以直接输出到elasticsearchoutput.logstash: hosts: ["192.168.1.30:5044"]tls: certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key" bulk_max_size: 1024logging.to_files: truelogging.files:#编写Logstash配置文件input { beats { host => "192.168.1.30" port => 5044 ssl => false ssl_certificate => "/etc/pki/tls/certs/logstash.crt" ssl_key => "/etc/pki/tls/private/logstash.key" ssl_certificate_authorities => ["/etc/pki/tls/certs/filebeat.crt"] ssl_verify_mode => "force_peer" }}filter { if [type] == "nginxlog" { grok { match => { "message" => "%{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:method} %{NOTSPACE:request}(?: %{URIPROTO:proto}/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:size}|-) %{QS:referrer}%{QS:agent} %{QS:xforwardedfor}" } add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{host}" ] } date { match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ] } }}output { if [type] == "nginxlog" { elasticsearch { hosts => ["192.168.1.30:9200"] index => "logstash-nginxlog-%{+YYYY.MM.dd}" manage_template => true flush_size => 50000 idle_flush_time => 10}}}访问elk.sudo.com，Index name or pattern输入logstash-nginxlog-*，Time-field name选择json@timestamp，，创建添加#上两种方式分别针对的是不可json化与可json化日志，对应的配置也有所不同，注意相关参数###############################################################################同时监控nginx/systemlog日志（filebeat输出到logstash，免filter存入elasticsearch,cd /usr/local/logstash-5.1.2vim config/logstash.confinput { beats { host => "192.168.1.30" port => 5044 ssl => false ssl_certificate => "/etc/pki/tls/certs/logstash.crt" ssl_key => "/etc/pki/tls/private/logstash.key" ssl_certificate_authorities => ["/etc/pki/tls/certs/filebeat.crt"] ssl_verify_mode => "force_peer" }}filter {}output { if [type] == "syslog" { elasticsearch { hosts => ["192.168.1.30:9200"] manage_template => false index => "filebeat-messages-%{+YYYY.MM.dd}" flush_size => 50000 idle_flush_time => 10 }} if [type] == "nginxlog" { elasticsearch { hosts => ["192.168.1.30:9200"] manage_template => false index => "filebeat-nginxlog-%{+YYYY.MM.dd}" flush_size => 50000 idle_flush_time => 10 }} if [type] == "secure" { elasticsearch { hosts => ["192.168.1.30:9200"] manage_template => false index => "filebeat-secure-%{+YYYY.MM.dd}" flush_size => 50000 idle_flush_time => 10 }}}#添加filebeat配置vim filebeat.ymlfilebeat.prospectors:- input_type: log paths: - /var/log/messages document_type: syslog- input_type: log paths: - /var/log/nginx/access.log document_type: nginxlog- input_type: log paths: - /var/log/secure document_type: secureoutput.elasticsearch: hosts: ["192.168.1.30:5043"]##不直接发送到elasticsearch，因nginx日志格式已经json化，可以直接输出到elasticsearchoutput.logstash: hosts: ["192.168.1.30:5044"]tls: certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key" bulk_max_size: 1024logging.to_files: truelogging.files:访问elk.sudo.com，Index name or pattern分别输入logstash-messages-*，Time-field name选择@timestamp，创建添加#同时监控nginx/systemlog日志（filebeat直接输出存入elasticsearch）无需配置logstash，直接在filebeat端配置output.elasticsearch参数#添加filebeat配置vim filebeat.ymlfilebeat.prospectors:- input_type: log paths: - /var/log/messages document_type: syslog- input_type: log paths: - /var/log/nginx/access.log document_type: nginxlog- input_type: log paths: - /var/log/secure document_type: secureoutput.elasticsearch: hosts: ["192.168.1.30:9200"] index: "filebeat-systemlog-%{+yyyy.MM.dd}"#output.logstash: #hosts: ["192.168.1.30:5044"]#直接输入到发送到elasticsearch，不通过logstashtls: certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"] certificate: "/etc/pki/tls/certs/filebeat.crt" certificate_key: "/etc/pki/tls/private/filebeat.key" bulk_max_size: 1024logging.to_files: truelogging.files:访问elk.sudo.com，Index name or pattern输入filbeat-systemlog-*，Time-field name选择@timestamp，，创建添加###############################################################################遇到的问题：################################################################################Java HotSpot(TM) 64-Bit Server VM warning: INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) failed; error='Cannot allocate memory' (errno=12)由于elasticsearch5.x默认分配jvm空间大小为2g，修改jvm空间分配# vim config/jvm.options -Xms2g -Xmx2g 修改为# vim config/jvm.options -Xms512m -Xmx512m#max file descriptors [65535] for elasticsearch process is too low, increase to at least [65536]ulimit -SHn 65536vim /etc/security/limits.conf* soft nofile 65536* hard nofile 65536* soft nproc 65536* hard nproc 65536* soft nofile 65536* hard nofile 65536* soft nproc 65536* hard nproc 65536* soft nofile 65536* hard nofile 65536#max number of threads [1024] for user [elasticsearch] is too low, increase to at least [2048]修改 /etc/security/limits.d/90-nproc.conf * soft nproc 1024* soft nproc 2048#max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]修改/etc/sysctl.conf配置文件，cat /etc/sysctl.conf | grep vm.max_map_countvm.max_map_count=262144如果不存在则添加echo "vm.max_map_count=262144" >>/etc/sysctl.conf