1、服务器基础配置
服务器配置
系统
相关服务
4VCPU8G
Centos72
Zookeeper、hadoop、flink
4VCPU8G
Centos72
Zookeeper、hadoop、flink
4VCPU8G
Centos72
Zookeeper、hadoop、flink
1.1、Node01节点
[root@localhost ~]# yum update -y[root@localhost ~]# yum install vim net-toolsntpdate[root@localhost ~]# hostnamectl set-hostname node01[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld[root@localhost ~]# iptables -F[root@localhost ~]# vim /etc/selinux/configSELINUX=disabled[root@localhost ~]# vim /etc/hosts192.168.2.121 node01192.168.2.122 node02192.168.2.123 node03[root@node01 ~]# crontab -e*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com[root@localhost ~]# useradd clu_flink[root@localhost ~]# su clu_flink[clu_flink@node01 root]$ cd ~[clu_flink@node01 ~]$ ssh-keygen -t rsa -P ” -f ~/.ssh/id_rsaGenerating public/private rsa key pair.Created directory ‘/home/clu_flink/.ssh’.Your identification has been saved in /home/clu_flink/.ssh/id_rsa.Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.The key fingerprint is:SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node01The key’s randomart image is:+—[RSA 2048]—-+| ..oo=B*o*|| . oO.++ +*|| +=.B o *=|| E + +B + + o|| . o . S* * || o . .. * || . . || || |+—-[SHA256]—–+[clu_flink@node01 ~]$ cd .ssh/[clu_flink@node01 .ssh]$ cp id_rsa.pub authorized_keys[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node01[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node02[clu_flink@node01 .ssh]$ ssh-copy-id clu_flink@node03[root@node01 flink]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/[root@node01 flink]# vim /etc/profileexport JAVA_HOME=/usr/local/jdk1.8.0_92/export PATH=$PATH:$JAVA_HOME/binexport CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar[root@node01 flink]# source /etc/profile[root@node01 flink]# java -versionjava version “1.8.0_92″Java(TM) SE Runtime Environment (build 1.8.0_92-b14)Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
1.2、Node02节点
[root@localhost ~]# yum update -y[root@localhost ~]# yum install vim net-toolsntpdate[root@localhost ~]# hostnamectl set-hostname node02[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld[root@localhost ~]# iptables -F[root@localhost ~]# vim /etc/selinux/configSELINUX=disabled[root@localhost ~]# vim /etc/hosts192.168.2.121 node01192.168.2.122 node02192.168.2.123 node03[root@node02 ~]# crontab -e*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com[root@localhost ~]# useradd clu_flink[root@localhost ~]# su clu_flink[clu_flink@node02root]$ cd ~[clu_flink@node02~]$ ssh-keygen -t rsa -P ” -f ~/.ssh/id_rsaGenerating public/private rsa key pair.Created directory ‘/home/clu_flink/.ssh’.Your identification has been saved in /home/clu_flink/.ssh/id_rsa.Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.The key fingerprint is:SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node02The key’s randomart image is:+—[RSA 2048]—-+| ..oo=B*o*|| . oO.++ +*|| +=.B o *=|| E + +B + + o|| . o . S* * || o . .. * || . . || || |+—-[SHA256]—–+[clu_flink@node02~]$ cd .ssh/[clu_flink@node02.ssh]$ cp id_rsa.pub authorized_keys[clu_flink@node02.ssh]$ ssh-copy-id clu_flink@node01[clu_flink@node02 .ssh]$ ssh-copy-id clu_flink@node02[clu_flink@node02 .ssh]$ ssh-copy-id clu_flink@node03[root@node02 ~]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/[root@node02 ~]# vim /etc/profileexport JAVA_HOME=/usr/local/jdk1.8.0_92/export PATH=$PATH:$JAVA_HOME/binexport CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar[root@node02 ~]# source /etc/profile[root@node02 ~]# java -versionjava version “1.8.0_92″Java(TM) SE Runtime Environment (build 1.8.0_92-b14)Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
1.3、Node03节点
[root@localhost ~]# yum update -y[root@localhost ~]# yum install vim net-toolsntpdate[root@localhost ~]# hostnamectl set-hostname node03[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld[root@localhost ~]# iptables -F[root@localhost ~]# vim /etc/selinux/configSELINUX=disabled[root@localhost ~]# vim /etc/hosts192.168.2.121 node01192.168.2.122 node02192.168.2.123 node03[root@node03 ~]# crontab -e*/5 * * * * /usr/sbin/ntpdate ntp1.aliyun.com[root@localhost ~]# useradd clu_flink[root@localhost ~]# su clu_flink[clu_flink@node03 root]$ cd ~[clu_flink@node03 ~]$ ssh-keygen -t rsa -P ” -f ~/.ssh/id_rsaGenerating public/private rsa key pair.Created directory ‘/home/clu_flink/.ssh’.Your identification has been saved in /home/clu_flink/.ssh/id_rsa.Your public key has been saved in /home/clu_flink/.ssh/id_rsa.pub.The key fingerprint is:SHA256:FGWZO2HmdLLatFOsTBdR5ZxmlXE/PULJERyo8Cbjglw clu_flink@node03The key’s randomart image is:+—[RSA 2048]—-+| ..oo=B*o*|| . oO.++ +*|| +=.B o *=|| E + +B + + o|| . o . S* * || o . .. * || . . || || |+—-[SHA256]—–+[clu_flink@node03 ~]$ cd .ssh/[clu_flink@node03 .ssh]$ cp id_rsa.pub authorized_keys[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node01[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node02[clu_flink@node03 .ssh]$ ssh-copy-id clu_flink@node03[root@node03 ~]# tar -xf jdk1.8.0_92.tar.gz -C /usr/local/[root@node03 ~]# vim /etc/profileexport JAVA_HOME=/usr/local/jdk1.8.0_92/export PATH=$PATH:$JAVA_HOME/binexport CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar[root@node03 ~]# source /etc/profile[root@node03 ~]# java -versionjava version “1.8.0_92″Java(TM) SE Runtime Environment (build 1.8.0_92-b14)Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode)
2、Zookeeper集群2.1、zookeeper概述
ZooKeeper是一种为分布式应用所设计的高可用、高性能且一致的开源协调服务，它提供一项基本服务：分布式锁服务。
分布式应用可以基于它实现更高级的服务，实现诸如同步服务、配置维护和集群管理或者命名的服务。Zookeeper服务自身组成一个集群，2n+1个（奇数）服务允许n个失效，集群内一半以上机器可用，Zookeeper就可用。
假设3台机器组成的集群，可以有允许一台失效，如果有2台失效，这个集群就不可用，1<1.5，一般的搭建zookeeper集群时，以奇数台机器来搭建。目的：是为了提高容错能允许多损失一台。
Leader主要有三个功能：
1.恢复数据；
2.维持与follower的心跳，接收follower请求并判断follower的请求消息类型；
3.follower的消息类型主要有PING消息、REQUEST消息、ACK消息、REVALIDATE消息，根据不同的消息类型，进行不同的处理。
Follower主要有四个功能：
1.向Leader发送请求（PING消息、REQUEST消息、ACK消息、REVALIDATE消息）；
2.接收Leader消息并进行处理；
3.接收Client的请求，如果为写请求，发送给Leader进行投票；
4.返回Client结果。
读操作分析：
ZooKeeper集群中所有的server节点都拥有相同的数据，所以读的时候可以在任意一台server节点上，客户端连接到集群中某一节点，读请求，然后直接返回。当然因为ZooKeeper协议的原因（一半以上的server节点都成功写入了数据，这次写请求便算是成功），读数据的时候可能会读到数据不是最新的server节点，所以比较推荐使用watch机制，在数据改变时，及时感应到。
写操作分析：
当一个客户端进行写数据请求时，会指定ZooKeeper集群中的一个server节点，如果该节点为Follower，则该节点会把写请求转发给Leader，Leader通过内部的协议进行原子广播，直到一半以上的server节点都成功写入了数据，这次写请求便算是成功，然后Leader便会通知相应Follower节点写请求成功，该节点向client返回写入成功响应。
zookeeper 的三个端口作用：
2181 : 对 client 端提供服务
2888 : 集群内机器通信使用
3888 : 选举 leader 使用
2.2、Node01节点配置
[root@node01 flink]# mkdir /data[root@node01 flink]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/ [root@node01 flink]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper[root@node01 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg [root@node01 zookeeper]# vim /data/zookeeper/conf/zoo.cfg tickTime=2000initLimit=10syncLimit=5dataDir=/data/zookeeper/datadataLogDir=/data/zookeeper/logsclientPort=2181server.1=192.168.2.121:3188:3288server.2=192.168.2.122:3188:3288server.3=192.168.2.123:3188:3288[root@node01 zookeeper]# vim /etc/profile#zookeeperexport ZK_HOME=/data/zookeeperexport PATH=$PATH:$ZK_HOME/bin[root@node01 zookeeper]# source /etc/profile添加java环境变量[root@node01 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh JAVA_HOME=”/usr/local/jdk1.8.0_92/”
[root@node01 zookeeper]# vim /etc/init.d/zookeeper#!/bin/bashZK_HOME=’/data/zookeeper’ ZK_BIN=’bin/zkServer.sh’ case $1 in start) echo “———- zookeeper 启 动 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} start”;; stop) echo “———- zookeeper 停 止 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} stop”;; restart) echo “———- zookeeper 重 启 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} restart”;; status) echo “———- zookeeper 状 态 ————“sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} status”;; *) echo “Usage: $0 {start|stop|restart|status}” esac [root@node01 zookeeper]# chmod +x /etc/init.d/zookeeper[root@node01 ~]# echo 1 > /data/zookeeper/data/myid 这个ID存放绝对不能放错，放在数据目录下[root@node01 zookeeper]# chown clu_flink.clu_flink /data/zookeeper/ -R[root@node01 ~]# /etc/init.d/zookeeper start
2.3、Node02节点配置
[root@node02 ~]# mkdir /data[root@node02 ~]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/[root@node02 ~]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper[root@node02 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg [root@node02 ~]# vim /data/zookeeper/conf/zoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/data/zookeeper/datadataLogDir=/data/zookeeper/logsclientPort=2181server.1=192.168.2.121:3188:3288server.2=192.168.2.122:3188:3288server.3=192.168.2.123:3188:3288[root@node02 zookeeper]# vim /etc/profile#zookeeperexport ZK_HOME=/data/zookeeperexport PATH=$PATH:$ZK_HOME/bin[root@node01 zookeeper]# source /etc/profile添加java环境变量[root@node02 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh JAVA_HOME=”/usr/local/jdk1.8.0_92/”
[root@node02 ~]# vim /etc/init.d/zookeeper#!/bin/bashZK_HOME=’/data/zookeeper’ ZK_BIN=’bin/zkServer.sh’ case $1 in start) echo “———- zookeeper 启 动 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} start”;; stop) echo “———- zookeeper 停 止 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} stop”;; restart) echo “———- zookeeper 重 启 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} restart”;; status) echo “———- zookeeper 状 态 ————“sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} status”;; *) echo “Usage: $0 {start|stop|restart|status}” esac[root@node02 ~]# chmod +x /etc/init.d/zookeeper[root@node02 ~]# echo 2 > /data/zookeeper/data/myid[root@node02 ~]# chown clu_flink.clu_flink /data/zookeeper/ -R[root@node02 ~]# /etc/init.d/zookeeper start
2.4、Node03节点配置
[root@node03 ~]# mkdir /data[root@node03 ~]# tar -xf apache-zookeeper-3.8.0-bin.tar.gz -C /data/[root@node03 ~]# mv /data/apache-zookeeper-3.8.0-bin/ /data/zookeeper[root@node03 ~]# cp /data/zookeeper/conf/zoo_sample.cfg /data/zookeeper/conf/zoo.cfg [root@node03 ~]# vim /data/zookeeper/conf/zoo.cfgtickTime=2000initLimit=10syncLimit=5dataDir=/data/zookeeper/datadataLogDir=/data/zookeeper/logsclientPort=2181server.1=192.168.2.121:3188:3288server.2=192.168.2.122:3188:3288server.3=192.168.2.123:3188:3288[root@node03 zookeeper]# vim /etc/profile#zookeeperexport ZK_HOME=/data/zookeeperexport PATH=$PATH:$ZK_HOME/bin[root@node01 zookeeper]# source /etc/profile添加java环境变量[root@node03 zookeeper]# vim /data/zookeeper/bin/zkEnv.sh JAVA_HOME=”/usr/local/jdk1.8.0_92/”
[root@node03 ~]# vim /etc/init.d/zookeeper#!/bin/bashZK_HOME=’/data/zookeeper’ ZK_BIN=’bin/zkServer.sh’ case $1 in start) echo “———- zookeeper 启 动 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} start”;; stop) echo “———- zookeeper 停 止 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} stop”;; restart) echo “———- zookeeper 重 启 ————” sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} restart”;; status) echo “———- zookeeper 状 态 ————“sudo -H -u clu_flink bash -c “cd ${ZK_HOME} && ${ZK_BIN} status”;; *) echo “Usage: $0 {start|stop|restart|status}” esac[root@node03 ~]# chmod +x /etc/init.d/zookeeper[root@node03 ~]# echo 3 > /data/zookeeper/data/myid[root@node03 ~]# chown clu_flink.clu_flink /data/zookeeper/ -R[root@node03 ~]# /etc/init.d/zookeeper start
以上zookeeper集群搭建完成，可以通过以下命令进行检查??zkCli.sh?? -server 192.168.2.123:2181 记得更换相关节点IP即可
学会宽容，要有一颗宽容的爱心！
