版本
elk版本是7.3.2
jdk大于1.8
shell脚本
#/bin/bash
#Author: hanzhenyong
#此脚本为ES单节点版
#此脚本和安装包在同一路径下
#创建esuser用户
if id -u esuser >/dev/null 2>&1; then
echo "esuser exists"
else
echo "esuer does not exist" && echo "创建用户" && useradd esuser && echo "用户创建完成"
fi
#创建数据目录并赋权
read -p "please input The data directory (example: /data01,/data02):" data
#获取长度
num=`echo $data|awk -F',' '{line=NF}END{print line}'`
for((i=1;i<=$num;i++))
do
mkdir -pv `echo $data|awk -F',' -v t=$i '{print $t}'`/elasticsearch/data
chown -R esuser.esuser `echo $data|awk -F',' -v t=$i '{print $t}'`/elasticsearch/data
done
mkdir -pv /opt/elk/esuser
mkdir -pv /var/log/elk/elasticsearch
mkdir -pv /var/log/elk/kibana
#解压软件
DIR=$(cd $(dirname $0) && pwd )
tar -zxvf $DIR/elasticsearch-7.3.2-linux-x86_64.tar.gz -C /opt/elk/esuser/ >/dev/null 2>&1
tar -zxvf $DIR/elasticsearch-head.tar.gz -C /opt/elk/esuser/ >/dev/null 2>&1
tar -zxvf $DIR/kibana-7.3.2-linux-x86_64.tar.gz -C /opt/elk/esuser/ >/dev/null 2>&1
tar -zxvf $DIR/node-v6.2.2-linux-x64.tar.gz -C /opt/elk/esuser/ >/dev/null 2>&1
tar -zxvf $DIR/logstash-7.3.2.tar.gz -C /opt/elk/esuser/ >/dev/null 2>&1
#安装分词器
unzip $DIR/elasticsearch-analysis-ik-7.3.2.zip -d /opt/elk/esuser/elasticsearch-7.3.2/plugins/analysis-ik >/dev/null 2>&1
#创建软连接
ln -s /opt/elk/esuser/elasticsearch-7.3.2/ /opt/elk/esuser/elasticsearch
ln -s /opt/elk/esuser/kibana-7.3.2-linux-x86_64/ /opt/elk/esuser/kibana
ln -s /opt/elk/esuser/node-v6.2.2-linux-x64/ /opt/elk/esuser/node
ln -s /opt/elk/esuser/logstash-7.3.2/ /opt/elk/esuser/logstash
#目录授权
chown -R esuser.esuser /opt/elk/esuser /var/log/elk/elasticsearch /var/log/elk/kibana
#修改用户的内存限制
cat << EOF >>/etc/security/limits.d/20-nproc.conf
esuser soft memlock unlimited
esuser hard memlock unlimited
EOF
#修改内核参数
i=`cat /etc/sysctl.conf |grep vm.max_map_count=655360`
if [ $? -eq 0 ];then
echo "vm.max_map_count=655360已存在" && sed -i "s/vm.max_map_count=655360/vm.max_map_count=262144/g" >>/etc/sysctl.conf
else
echo 'vm.max_map_count=655360不已存在' && echo 'vm.max_map_count=262144' >>/etc/sysctl.conf && echo 'vm.max_map_count=262144已添加'
fi
sysctl -p /etc/sysctl.conf >/dev/null 2>&1
read -p "please input The es_data directory (example: /data01/elasticsearch/data, /data02/elasticsearch/data):" es_data
read -p "please input The es_hostname (example: \"bigdatanode001.idatafusion.dsgaw.gov\"):" es_hostname
read -p "please input The es_hostname_port (example: \"bigdatanode001.idatafusion.dsgaw.gov:39300\"):" es_hostname_port
read -p "please input es_ip:" ip
read -p "please input ngip:" ngip
#elasticsearch配置
cp /opt/elk/esuser/elasticsearch/config/elasticsearch.yml /opt/elk/esuser/elasticsearch/config/elasticsearch.yml.bak
cat << EOF > /opt/elk/esuser/elasticsearch/config/elasticsearch.yml
cluster.name: es7.3.2
node.name: \${HOSTNAME}
path.data: $es_data
path.logs: /var/log/elk/elasticsearch
bootstrap.memory_lock: true
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 39200
transport.tcp.port: 39300
network.bind_host: 0.0.0.0
network.publish_host: $ip
discovery.zen.ping.unicast.hosts: ["$es_hostname_port"]
cluster.initial_master_nodes: ["$es_hostname"]
action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml*,elk_log*
xpack.security.enabled: false
thread_pool.search.queue_size: 2000
indices.memory.index_buffer_size: 40%
indices.recovery.max_bytes_per_sec: 50mb
action.destructive_requires_name: true
EOF
chown -R esuser.esuser /opt/elk/esuser/elasticsearch/config/elasticsearch.yml
#内存设置
read -p "please input The es_mem (example: 31g):" es_mem
sed -i 's/-Xms1g/-Xms'$es_mem'/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/-Xmx1g/-Xmx'$es_mem'/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/-XX:+UseConcMarkSweepGC/#-XX:+UseConcMarkSweepGC/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/-XX:CMSInitiatingOccupancyFraction=75/#-XX:CMSInitiatingOccupancyFraction=75/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/-XX:+UseCMSInitiatingOccupancyOnly/#-XX:+UseCMSInitiatingOccupancyOnly/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/# 10-:-XX:-UseConcMarkSweepGC/10-:-XX:-UseConcMarkSweepGC/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/# 10-:-XX:-UseCMSInitiatingOccupancyOnly/10-:-XX:-UseCMSInitiatingOccupancyOnly/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/# 10-:-XX:+UseG1GC/10-:-XX:+UseG1GC/g' /opt/elk/esuser/elasticsearch/config/jvm.options
sed -i 's/# 10-:-XX:InitiatingHeapOccupancyPercent=75/10-:-XX:InitiatingHeapOccupancyPercent=75/g' /opt/elk/esuser/elasticsearch/config/jvm.options
#kibana配置
#ip=$(ip a |grep 'scope global' |awk '{print $2}'|awk -F "/" '{print $1}'|awk NR==2)
#read -p "please input es_ip:" ip
cp /opt/elk/esuser/kibana/config/kibana.yml /opt/elk/esuser/kibana/config/kibana.yml.bak
cat << EOF > /opt/elk/esuser/kibana/config/kibana.yml
server.port: 35601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:39200"]
pid.file: /home/esuser/kibana.pid
logging.dest: /var/log/elk/kibana/kibana.log
logging.quiet: true
i18n.locale: "zh-CN"
EOF
#head配置
sed -i "s/10.19.120.67:39200/$ip/g" /opt/elk/esuser/elasticsearch-head/_site/app.js
#修改环境变量
cat << EOF >> /home/esuser/.bash_profile
export PUBIP=$ip
export NODE_HOME=/opt/elk/esuser/node
export ES_HOME=/opt/elk/esuser/elasticsearch
export JAVA_HOME=\$ES_HOME/jdk
export PATH=\$JAVA_HOME/bin:\$ES_HOME/bin:\$NODE_HOME/bin:\$PATH
EOF
#chmod 777 /opt/elk/esuser/elasticsearch-head/node_modules/grunt/bin/grunt
chmod 777 /opt/elk/esuser/elasticsearch-head/node_modules/grunt/bin/grunt
chmod 777 /opt/elk/esuser/node/bin/*
#Logstash配置
#默认配置nginx监控
echo "#Nginx log
WZ([^]*)
NGINXACCESS %{IP:remote_jp} \- \- \[%{HTTPDATE:timestamp}\] "%{WORD:method}% {WZ:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:bytes} %{QS:referer} %{QS:agent} %{QS:xforward}" >> /opt/elk/esuser/logstash-7.3.2/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns/grok-patterns
#创建logstash配置文件
cat << EOF >> /opt/elk/esuser/logstash-7.3.2/default.conf
input {
beats {
port => "5044"
}
}
# 数据过滤
filter {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
geoip {
# nginx客户端ip
source => "$ngip"
}
}
# 输出配置为本机的9200端口，这是ElasticSearch服务的监听端口
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
}
}
EOF
#添加logstash启动脚本
#添加logstash自启动脚本
cat << EOF > /etc/init.d/logstash
#!/bin/bash
# chkconfig: 2345 20 70
# description: logstash Service
RUNAS=root
LOGSTASH_HOME=/opt/elk/esuser/logstash-7.3.2
#EXEC=\$LOGSTASH_HOME/bin/logstash
#PIDFILE=\$(sed '/^pid.file:/!d;s/^pid.file: *//' \$LOGSTASH_HOME/config/startup.options)
PIDFILE=/var/run/logstash.pid
#if [ ! -x \$EXEC ] ; then
# echo "ERROR: Permission denied or \$EXEC not found"
# exit 1
#fi
start(){
# verify the specified run as user exists
/opt/elk/esuser/logstash-7.3.2/bin/logstash -f /opt/elk/esuser/logstash-7.3.2/default.conf & >> 2 >& 1 >> /dev/null
}
stop(){
if [ ! -f \$PIDFILE ]
then
echo "\$PIDFILE does not exist, process is not running"
else
PID=\$(cat \$PIDFILE)
kill \$PID
while [ -d /proc/\${PID} ]
do
echo "Waiting for logstash to shutdown ..."
sleep 1
done
echo "logstash stopped!"
fi
}
restart(){
stop
start
}
status(){
if [ -f \$PIDFILE ] ;then
echo "logstash have already started!"
else
echo "logstash have already stoped!"
fi
}
case "\$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
status)
status
;;
*)
echo "Usage: service logstash {start|stop|restart|status}"
;;
esac
EOF
#添加elasticsearch自启动脚本
cat << EOF >/etc/init.d/elasticsearch
#!/bin/bash
# chkconfig: 2345 20 70
# description: Elasticsearch Service
RUNAS=esuser
ES_HOME=/opt/elk/esuser/elasticsearch
EXEC=\$ES_HOME/bin/elasticsearch
PIDFILE=/home/\$RUNAS/elasticsearch.pid
if [ ! -x \$EXEC ] ; then
echo "ERROR: Permission denied or \$EXEC not found"
exit 1
fi
start(){
# verify the specified run as user exists
id \${RUNAS} >/dev/null 2>&1
if [ \$? -ne 0 ]; then
echo "User \${RUNAS} not found! Please create the \${RUNAS} user before running this script."
exit 1
fi
if [ -f \$PIDFILE ]
then
PID=\$(cat \$PIDFILE)
if [ -d /proc/\${PID} ]
then
echo "\${PIDFILE} exists, process is already running or crashed"
else
rm -f \${PIDFILE}
runuser -c "\$EXEC --pidfile \$PIDFILE --daemonize" - \${RUNAS:-esuser}
fi
else
runuser -c "\$EXEC --pidfile \$PIDFILE --daemonize" - \${RUNAS:-esuser}
while [ -f \$PIDFILE ]
do
echo "Waiting for Elasticsearch to start ..."
sleep 1
done
echo "Elasticsearch running!"
fi
}
stop(){
if [ ! -f \$PIDFILE ]
then
echo "\$PIDFILE does not exist, process is not running"
else
PID=\$(cat $PIDFILE)
kill \$PID
while [ -d /proc/\${PID} ]
do
echo "Waiting for Elasticsearch to shutdown ..."
sleep 1
done
echo "Elasticsearch stopped!"
fi
}
restart(){
stop
start
}
status(){
if [ -f \$PIDFILE ] ;then
echo "Elasticsearch have already started!"
else
echo "Elasticsearch have already stoped!"
fi
}
case "\$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
status)
status
;;
*)
echo "Usage: service elasticsearch {start|stop|restart|status}"
;;
esac
EOF
#添加elasticsearch-head自启动脚本
cat << EOF >/etc/init.d/elasticsearch-head
#!/bin/bash
# chkconfig: 2345 21 69
# description: Elasticsearch Service
export ServiceName="elasticsearch-head"
export NODE_HOME=/opt/elk/esuser/node
export PATH=\$PATH:$NODE_HOME/bin
start() {
su - esuser -c "/opt/elk/esuser/elasticsearch-head/node_modules/grunt/bin/grunt --gruntfile /opt/elk/esuser/elasticsearch-head/Gruntfile.js server &"
if [ \$? -ne 0 ] ;then exit 1;fi
sleep 2
echo
sleep 1
echo "\$ServiceName has been started..."
}
stop() {
killall -9 grunt
echo
echo "\$ServiceName has been stopped..."
}
case "\$1" in
start)
start
;;
stop)
stop
;;
*)
echo "Usage: service \$0 {start|stop}"
exit 1
esac
EOF
#添加kibana自启动脚本
cat << EOF > /etc/init.d/kibana
#!/bin/bash
# chkconfig: 2345 20 70
# description: Kibana Service
RUNAS=esuser
KIBANA_HOME=/opt/elk/esuser/kibana
EXEC=\$KIBANA_HOME/bin/kibana
PIDFILE=\$(sed '/^pid.file:/!d;s/^pid.file: *//' \$KIBANA_HOME/config/kibana.yml)
if [ ! -x \$EXEC ] ; then
echo "ERROR: Permission denied or \$EXEC not found"
exit 1
fi
start(){
# verify the specified run as user exists
id \${RUNAS} >/dev/null 2>&1
if [ \$? -ne 0 ]; then
echo "User \${RUNAS} not found! Please create the \${RUNAS} user before running this script."
exit 1
fi
if [ -f \$PIDFILE ]
then
PID=\$(cat \$PIDFILE)
if [ -d /proc/\${PID} ]
then
echo "\${PIDFILE} exists, process is already running or crashed"
else
rm -f \${PIDFILE}
runuser -c "\$EXEC &" - \${RUNAS:-esuser}
fi
else
runuser -c "\$EXEC &" - \${RUNAS:-esuser}
while [ -f \$PIDFILE ]
do
echo "Waiting for Kibana to start ..."
sleep 1
done
echo "Kibana running!"
fi
}
stop(){
if [ ! -f \$PIDFILE ]
then
echo "\$PIDFILE does not exist, process is not running"
else
PID=\$(cat \$PIDFILE)
kill \$PID
while [ -d /proc/\${PID} ]
do
echo "Waiting for Kibana to shutdown ..."
sleep 1
done
echo "Kibana stopped!"
fi
}
restart(){
stop
start
}
status(){
if [ -f \$PIDFILE ] ;then
echo "Kibana have already started!"
else
echo "Kibana have already stoped!"
fi
}
case "\$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
status)
status
;;
*)
echo "Usage: service kibana {start|stop|restart|status}"
;;
esac
EOF
chmod 755 /etc/init.d/elasticsearch /etc/init.d/elasticsearch-head /etc/init.d/kibana
#添加到系统服务启动列表
chkconfig --add elasticsearch
chkconfig --add elasticsearch-head
chkconfig --add kibana
chkconfig --add logstash
chkconfig elasticsearch on
chkconfig logstash on
chkconfig elasticsearch-head on
chkconfig kibana on
systemctl daemon-reload
#启动
service elasticsearch start
service elasticsearch-head start
service kibana start
service logstash start