ansible test3

bigdata-scripts_test3/deploy.yml

@@ -0,0 +1,21 @@
+- hosts: server1
+  roles:
+    - jdk1.8.0_73
+    - zookeeper
+    - kafka
+
+- hosts: server2
+  roles:
+    - jdk1.8.0_73
+    - zookeeper
+    - kafka
+
+- hosts: server3
+  roles:
+    - jdk1.8.0_73
+    - zookeeper
+    - kafka
+
+- hosts: server1
+  roles:
+    - kafka-manager

bigdata-scripts_test3/roles/jdk1.8.0_73/files/set_java_enviroment

@@ -0,0 +1,5 @@
+#set java enviroment
+export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_73
+export JRE_HOME=$JAVA_HOME/jre
+export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib
+export PATH=$PATH:$JAVA_HOME/bin

bigdata-scripts_test3/roles/jdk1.8.0_73/tasks/main.yml

@@ -0,0 +1,31 @@
+- name: "copy JDK install files to destination server"
+  copy:
+    src: "{{ role_path }}/files/"
+    dest: /tmp
+    mode: 0755
+
+- name: "judge old java"
+  shell: rpm -qa | grep openjdk 
+  register: return
+  ignore_errors: true
+
+- name: "remove old java"
+  shell: rpm -qa | grep openjdk | xargs rpm -e --nodeps
+  when: return.rc == 0
+    
+- name: "install JDK1.8.0_73"
+  unarchive:
+    src: "/tmp/jdk-8u73-linux-x64.tar.gz"
+    dest: /usr/lib/jvm/
+    mode: 0755
+
+- name: "judge java enviroment"
+  shell: grep "/usr/lib/jvm/jdk1.8.0_73" /etc/profile
+  register: return
+
+- name: "set java enviroment"
+  shell: cat /tmp/set_java_enviroment >> /etc/profile
+  when: return.rc != 0
+
+- name: "source /etc/profile"
+  shell: source /etc/profile

bigdata-scripts_test3/roles/kafka-manager/tasks/main.yml

@@ -0,0 +1,35 @@
+- name: "copy kafka_manager install package to destination server"
+  copy:
+    src: "{{ role_path }}/files/{{ kafka_manager.package_name }}"
+    dest: /tmp
+
+- name: "install kafka_manager"
+  unarchive:
+    src: "/tmp/{{ kafka_manager.package_name }}"
+    dest: '{{ kafka.base_dir }}'
+    copy: no
+    mode: 0755
+
+- name: "template application.conf"
+  template:
+    src: "{{ role_path }}/templates/application.conf.j2"
+    dest: '{{ kafka.base_dir }}/{{ kafka_manager.version }}/conf/application.conf
+    mode: 0755
+
+- name: "template application.conf"
+  template:
+    src: "{{ role_path }}/templates/start.sh.j2"
+    dest: '{{ kafka.base_dir }}/{{ kafka_manager.version }}/start.sh'
+    mode: 0755
+
+- name: "start kafka_manager"
+  shell: /bin/bash {{ kafka.base_dir }}/{{ kafka_manager.version }}/start.sh
+
+- name: "template create-topic.sh"
+  template:
+    src: "{{ role_path }}/templates/create-topic.sh.j2"
+    dest: '{{ kafka.base_dir }}/{{ kafka.version }}/bin/create-topic.sh'
+    mode: 0755
+
+- name: "create topic"
+  shell: source /etc/profile;/bin/bash {{ kafka.base_dir }}/{{ kafka.version }}/bin/create-topic.sh 

bigdata-scripts_test3/roles/kafka-manager/templates/application.conf.j2

@@ -0,0 +1,47 @@
+
+# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
+# See accompanying LICENSE file.
+
+# This is the main configuration file for the application.
+# ~~~~~
+
+# Secret key
+# ~~~~~
+# The secret key is used to secure cryptographics functions.
+# If you deploy your application to several instances be sure to use the same key!
+play.crypto.secret="^<csmm5Fx4d=r2HEX8pelM3iBkFVv?k[mc;IZE<_Qoq8EkX_/7@Zt6dP05Pzea3U"
+play.crypto.secret=${?APPLICATION_SECRET}
+
+# The application languages
+# ~~~~~
+play.i18n.langs=["en"]
+
+play.http.requestHandler = "play.http.DefaultHttpRequestHandler"
+play.http.context = "/"
+play.application.loader=loader.KafkaManagerLoader
+
+kafka-manager.zkhosts="{{ zookeeper.iplist[0] }}:2181"
+kafka-manager.zkhosts=${?ZK_HOSTS}
+pinned-dispatcher.type="PinnedDispatcher"
+pinned-dispatcher.executor="thread-pool-executor"
+application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"]
+
+akka {
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  loglevel = "INFO"
+}
+
+akka.logger-startup-timeout = 60s
+
+basicAuthentication.enabled=false
+basicAuthentication.enabled=${?KAFKA_MANAGER_AUTH_ENABLED}
+basicAuthentication.username="admin"
+basicAuthentication.username=${?KAFKA_MANAGER_USERNAME}
+basicAuthentication.password="password"
+basicAuthentication.password=${?KAFKA_MANAGER_PASSWORD}
+basicAuthentication.realm="Kafka-Manager"
+basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification
+
+kafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE}
+
+http.port=\"9998\"

bigdata-scripts_test3/roles/kafka-manager/templates/create-topic.sh.j2

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' CONNECTION-RECORD-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' PROXY-EVENT-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' RADIUS-RECORD-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' SECURITY-EVENT-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' CONNECTION-RECORD-COMPLETED-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' PROXY-EVENT-COMPLETED-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' RADIUS-RECORD-COMPLETED-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' SECURITY-EVENT-COMPLETED-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TRAFFIC-METRICS-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' SYS-DATA-STORAGE-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' SYS-STORAGE-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' PXY-EXCH-INTERMEDIA-CERT
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TOP-EXTERNAL-HOST-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TOP-INTERNAL-HOST-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TOP-USER-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TOP-URLS-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' TOP-WEBSITE-DOMAIN-LOG
+./kafka-topics.sh '--create' '--zookeeper' ''{{ zookeeper.iplist[0] }}':2181/kafka' '--replication-factor' 1 '--partitions' 3 '--topic' RADIUS-ONFF-LOG

bigdata-scripts_test3/roles/kafka-manager/templates/start.sh.j2

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+base_dir='{{ kafka.base_dir }}'
+
+if [ -f "$base_dir/kafka-manager-1.3.3.18/RUNNING_PID" ]; then 
+ rm -rf $base_dir/kafka-manager-1.3.3.18/RUNNING_PID
+fi
+
+nohup $base_dir/kafka-manager-1.3.3.18/bin/kafka-manager -Dconfig.file=$base_dir/kafka-manager-1.3.3.18/conf/application.conf > /dev/null 2>&1 &

bigdata-scripts_test3/roles/kafka/files/dae-kafka.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+#修改BASE_DIR为安装的路径
+#启动命令端口可自行指定
+#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 &
+
+PRO_NAME=Kafka
+BASE_DIR=$1
+VERSION="kafka_2.11-1.0.0"
+
+#string=`cat $BASE_DIR/$VERSION/config/server.properties | grep broker.id`
+#array=(${string//=/ })
+#echo ${array[1]}
+#ssh root@192.168.40.123 "source /etc/profile ; zkCli.sh ls /kafka/brokers/ids | grep -v 2181 | grep 4 | wc -l"
+source /etc/profile
+while true ; do
+    NUM=`jps | grep -w ${PRO_NAME} | grep -v grep |wc -l`
+    
+    if [ "${NUM}" -lt "1" ];then
+	JMX_PORT=9191 nohup $BASE_DIR/$VERSION/bin/kafka-server-start.sh $BASE_DIR/$VERSION/config/server.properties > /dev/null 2>&1 &
+        OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log`
+        RESTART_NUM=`expr $OLD_NUM + 1`
+        echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log
+        echo "`date "+%Y-%m-%d %H:%M:%S"` - kafka服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log
+	#大于1，杀掉所有进程，重启
+    elif [ "${NUM}" -gt "1" ];then
+        killall -9 ${PRO_NAME}
+    fi
+	sleep 60
+done

bigdata-scripts_test3/roles/kafka/files/kflogdelete.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+#只保留最近三天的日志，如需要多保留几天修改8行最后 -n days
+#将此脚本加载到系统定时任务中 /etc/crontab
+#脚本会读取环境变量，固需要配置环境变量。
+#. /etc/profile
+
+day=$(date  +"%Y-%m-%d" -d  "-3 days")
+
+kafka=`jps | grep Kafka | wc -l`
+if [[ $kafka = "1" ]];then
+    rm -rf $KAFKA_HOME/logs/*.$day*
+fi
+

bigdata-scripts_test3/roles/kafka/files/log4j.properties

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unspecified loggers and loggers with additivity=true output to server.log and stdout
+# Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
+log4j.rootLogger=INFO, stdout, kafkaAppender
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
+log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
+log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
+log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
+log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd
+log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
+log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Change the two lines below to adjust ZK client logging
+log4j.logger.org.I0Itec.zkclient.ZkClient=INFO
+log4j.logger.org.apache.zookeeper=INFO
+
+# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
+log4j.logger.kafka=INFO
+log4j.logger.org.apache.kafka=INFO
+
+# Change to DEBUG or TRACE to enable request logging
+log4j.logger.kafka.request.logger=WARN, requestAppender
+log4j.additivity.kafka.request.logger=false
+
+# Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output
+# related to the handling of requests
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
+#log4j.additivity.kafka.server.KafkaApis=false
+#log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
+log4j.logger.kafka.network.RequestChannel$=INFO, requestAppender
+log4j.additivity.kafka.network.RequestChannel$=false
+
+#log4j.logger.kafka.controller=TRACE, controllerAppender
+log4j.logger.kafka.controller=INFO, controllerAppender
+log4j.additivity.kafka.controller=false
+
+log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
+log4j.additivity.kafka.log.LogCleaner=false
+
+#log4j.logger.state.change.logger=TRACE, stateChangeAppender
+log4j.logger.state.change.logger=INFO, stateChangeAppender
+log4j.additivity.state.change.logger=false
+
+# Access denials are logged at INFO level, change to DEBUG to also log allowed accesses
+log4j.logger.kafka.authorizer.logger=INFO, authorizerAppender
+log4j.additivity.kafka.authorizer.logger=false
+

bigdata-scripts_test3/roles/kafka/tasks/main.yml

@@ -0,0 +1,74 @@
+- name: "copy kafka install package to destination server"
+  copy:
+    src: "{{ role_path }}/files/{{ kafka.package_name }}"
+    dest: /tmp
+
+- name: "create kafka base_dir"
+  file:
+    path: '{{ kafka.base_dir }}'
+    state: directory
+
+- name: "install kafka"
+  unarchive:
+    src: "/tmp/{{ kafka.package_name }}"
+    dest: '{{ kafka.base_dir }}'
+    copy: no
+    mode: 0755
+
+- name: "create logs dir"
+  file:
+    path: '{{ item.path }}'
+    state: '{{ item.state }}'
+  with_items:
+    - { path: '{{ kafka.base_dir }}/{{ kafka.version }}/logs', state: directory }
+    - { path: '{{ kafka.base_dir }}/{{ kafka.version }}/kafka-logs', state: directory }
+
+- name: "copy any scripts and log4j.properties"
+  copy:
+    src: '{{ item.src }}'
+    dest: '{{ item.dest }}'
+  with_items:
+    - { src: '{{ role_path }}/files/dae-kafka.sh', dest: '{{ kafka.base_dir }}/{{ kafka.version }}/bin/' }
+    - { src: '{{ role_path }}/files/kflogdelete.sh', dest: '{{ kafka.base_dir }}/{{ kafka.version }}/log/' }
+    - { src: '{{ role_path }}/files/log4j.properties', dest: '{{ kafka.base_dir }}/{{ kafka.version }}/config/' }
+    mode: 0755
+
+- name: "echo 0 > restart_sum.log"
+  shell: echo 0 > '{{ kafka.base_dir }}/{{ kafka.version }}/logs/restart_sum.log'
+
+- name: "template server.properties"
+  template:
+    src: "{{ role_path }}/templates/server.properties.j2"
+    dest: '{{ kafka.base_dir }}/{{ kafka.version }}/config/server.properties'
+
+- name: "template kafka-server-start.sh"
+  template:
+    src: "{{ role_path }}/templates/kafka-server-start.sh.j2"
+    dest: '{{ kafka.base_dir }}/{{ kafka.version }}/bin/kafka-server-start.sh'
+
+- name: "template keepkafalive"
+  template:
+    src: "{{ role_path }}/templates/keepkafalive.j2"
+    dest: /etc/init.d/keepkafalive.j2
+    mode: 0755
+
+- name: "start keepkafalive"
+  service:
+    name: keepkafalive
+    state: restarted
+    enabled: true
+
+- name: "template set_kafka_profile.sh"
+  template:
+    src: "{{ role_path }}/templates/set_kafka_profile.sh.j2"
+    dest: /tmp/set_kafka_profile.sh
+    mode: 0755
+
+- name: "judge kafka enviroment"
+  shell: grep "#kafka" /etc/profile
+  register: return
+  ignore_errors: true
+
+- name: "set_kafka_profile"
+  shell: /bin/bash /tmp/set_kafka_profile.sh
+  when: return.rc != 0

bigdata-scripts_test3/roles/kafka/templates/kafka-server-start.sh.j2

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+	echo "USAGE: $0 [-daemon] server.properties [--override property=value]*"
+	exit 1
+fi
+base_dir=$(dirname $0)
+
+if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
+    export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
+fi
+
+if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
+    export KAFKA_HEAP_OPTS="-Xmx{{ kafka.mem_max }} -Xms{{ kafka.mem_min }}"
+fi
+
+EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'}
+
+COMMAND=$1
+case $COMMAND in
+  -daemon)
+    EXTRA_ARGS="-daemon "$EXTRA_ARGS
+    shift
+    ;;
+  *)
+    ;;
+esac
+
+exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka "$@"

bigdata-scripts_test3/roles/kafka/templates/keepkafalive.j2

@@ -0,0 +1,11 @@
+#!/bin/bash
+# 
+# netconsole    This loads the netconsole module with the configured parameters.
+#
+# chkconfig:123456 30 70
+# description: keepkafkaalive
+source /etc/profile
+PRO_NAME=keepkafkaalive
+
+killall -9 dae-kafka.sh
+{{ kafka.base_dir }}/{{ kafka.version }}/bin/dae-kafka.sh {{ kafka.base_dir }} > /dev/null 2>&1 &

bigdata-scripts_test3/roles/kafka/templates/server.properties.j2

@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# see kafka.server.KafkaConfig for additional details and defaults
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id={{ broker-id }}
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. It will get the value returned from 
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+
+# Hostname and port the broker will advertise to producers and consumers. If not set, 
+# it uses the value for "listeners" if configured.  Otherwise, it will use the value
+# returned from java.net.InetAddress.getCanonicalHostName().
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+
+# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
+#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# The number of threads that the server uses for receiving requests from the network and sending responses to the network
+num.network.threads=3
+
+# The number of threads that the server uses for processing requests, which may include disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=10485760
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=10485760
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma seperated list of directories under which to store log files
+log.dirs={{ kafka.base_dir }}/{{ kafka.version }}/kafka-logs/
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=1
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Internal Topic Settings  #############################
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3.
+offsets.topic.replication.factor={{ kafka.replica }}
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+#    1. Durability: Unflushed data may be lost if you are not using replication.
+#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion due to age
+log.retention.hours={{ kafka.log_retentionHours }}
+
+# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
+log.retention.bytes={{ kafka.log_retentionBytes }}
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms={{ kafka.log_retentionCheckInterval_ms }}
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect={{ zookeeper.iplist[0] }}:2181,{{ zookeeper.iplist[1] }}:2181,{{ zookeeper.iplist[2] }}:2181/kafka
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=30000
+
+
+############################# Group Coordinator Settings #############################
+
+# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
+# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
+# The default value for this is 3 seconds.
+# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
+# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
+group.initial.rebalance.delay.ms=0
+
+
+#######################################################################################
+listeners=PLAINTEXT://{{ vars["ansible_"+kafka.ehtname]['ipv4']['address'] }}:9092
+advertised.listeners=PLAINTEXT://{{ vars["ansible_"+kafka.ethname]['ipv4']['address'] }}:9092
+#kafka port
+port=9092
+#Is it deleted directlytopic
+delete.topic.enable=true
+#Are you allowed to create automatically topic
+auto.create.topics.enable=false
+#Enable log periodic deletion strategy
+log.cleanup.policy=delete
+#The maximum size of a message body, unit byte.
+message.max.bytes=10485760
+#replicas Maximum size of data obtained eachtime
+replica.fetch.max.bytes=20485760
+

bigdata-scripts_test3/roles/kafka/templates/set_kafka_profile.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+echo -e "\n#kafka" >> /etc/profile
+echo -e "export KAFKA_HOME={{ kafka.base_dir }}/{{ kafka.version }}" >> /etc/profile
+echo -e "export PATH=\$KAFKA_HOME/bin:\$PATH" >> /etc/profile
+source /etc/profile

bigdata-scripts_test3/roles/zookeeper/files/dae-zookeeper.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+#进程名称
+PRO_NAME=QuorumPeerMain
+#安装路径
+BASE_DIR=$1
+VERSION="zookeeper-3.4.9"
+source /etc/profile
+while true ; do
+    NUM=`$BASE_DIR/$VERSION/bin/zkServer.sh status | egrep 'leader|follower' | wc -l`
+    if [ "${NUM}" -lt "1" ];then
+	$BASE_DIR/$VERSION/bin/zkServer.sh start
+        OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log`
+        RESTART_NUM=`expr $OLD_NUM + 1`
+        echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log
+        echo "`date "+%Y-%m-%d %H:%M:%S"` - zookeeper服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log
+	#大于1，杀掉所有进程，重启
+    elif [ "${NUM}" -gt "1" ];then
+        killall-9 ${PRO_NAME}
+    fi
+	sleep 60
+done

bigdata-scripts_test3/roles/zookeeper/files/log4j.properties

@@ -0,0 +1,63 @@
+# Define some default values that can be overridden by system properties
+#zookeeper.root.logger=INFO, CONSOLE
+zookeeper.root.logger=INFO, ROLLINGFILE
+zookeeper.console.threshold=INFO
+zookeeper.log.dir=.
+zookeeper.log.file=zookeeper.log
+zookeeper.log.threshold=DEBUG
+zookeeper.tracelog.dir=.
+zookeeper.tracelog.file=zookeeper_trace.log
+
+#
+# ZooKeeper Logging Configuration
+#
+
+# Format is "<default threshold> (, <appender>)+
+
+# DEFAULT: console appender only
+log4j.rootLogger=${zookeeper.root.logger}
+
+# Example with rolling log file
+#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
+
+# Example with rolling log file and tracing
+#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+#
+# Add ROLLINGFILE to rootLogger to get log file output
+#    Log DEBUG level and above messages to a log file
+#log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
+log4j.appender.ROLLINGFILE=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold}
+log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
+log4j.appender.ROLLINGFILE.DataPattern='.'yyyy-MM-dd-HH
+log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.ROLLINGFILE.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Max log file size of 10MB
+log4j.appender.ROLLINGFILE.MaxFileSize=10MB
+# uncomment the next line to limit number of backup files
+#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
+
+log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+
+#
+# Add TRACEFILE to rootLogger to get log file output
+#    Log DEBUG level and above messages to a log file
+log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
+log4j.appender.TRACEFILE.Threshold=TRACE
+log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file}
+
+log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
+### Notice we are including log4j's NDC here (%x)
+log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n

bigdata-scripts_test3/roles/zookeeper/files/restart_sum.log

@@ -0,0 +1 @@
+0

bigdata-scripts_test3/roles/zookeeper/files/zkEnv.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script should be sourced into other zookeeper
+# scripts to setup the env variables
+
+# We use ZOOCFGDIR if defined,
+# otherwise we use /etc/zookeeper
+# or the conf directory that is
+# a sibling of this script's directory
+
+ZOOBINDIR="${ZOOBINDIR:-/usr/bin}"
+ZOOKEEPER_PREFIX="${ZOOBINDIR}/.."
+
+if [ "x$ZOOCFGDIR" = "x" ]
+then
+  if [ -e "${ZOOKEEPER_PREFIX}/conf" ]; then
+    ZOOCFGDIR="$ZOOBINDIR/../conf"
+  else
+    ZOOCFGDIR="$ZOOBINDIR/../etc/zookeeper"
+  fi
+fi
+
+if [ -f "${ZOOCFGDIR}/zookeeper-env.sh" ]; then
+  . "${ZOOCFGDIR}/zookeeper-env.sh"
+fi
+
+if [ "x$ZOOCFG" = "x" ]
+then
+    ZOOCFG="zoo.cfg"
+fi
+
+ZOOCFG="$ZOOCFGDIR/$ZOOCFG"
+
+if [ -f "$ZOOCFGDIR/java.env" ]
+then
+    . "$ZOOCFGDIR/java.env"
+fi
+
+if [ "x${ZOO_LOG_DIR}" = "x" ]
+then
+    ZOO_LOG_DIR="${ZOOKEEPER_PREFIX}/logs/system"
+fi
+
+if [ "x${ZOO_LOG4J_PROP}" = "x" ]
+then
+   #ZOO_LOG4J_PROP="INFO,CONSOLE" 
+   ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
+fi
+
+if [ "$JAVA_HOME" != "" ]; then
+  JAVA="$JAVA_HOME/bin/java"
+else
+  JAVA=java
+fi
+
+#add the zoocfg dir to classpath
+CLASSPATH="$ZOOCFGDIR:$CLASSPATH"
+
+for i in "$ZOOBINDIR"/../src/java/lib/*.jar
+do
+    CLASSPATH="$i:$CLASSPATH"
+done
+
+#make it work in the binary package
+#(use array for LIBPATH to account for spaces within wildcard expansion)
+if [ -e "${ZOOKEEPER_PREFIX}"/share/zookeeper/zookeeper-*.jar ]; then
+  LIBPATH=("${ZOOKEEPER_PREFIX}"/share/zookeeper/*.jar)
+else
+  #release tarball format
+  for i in "$ZOOBINDIR"/../zookeeper-*.jar
+  do
+    CLASSPATH="$i:$CLASSPATH"
+  done
+  LIBPATH=("${ZOOBINDIR}"/../lib/*.jar)
+fi
+
+for i in "${LIBPATH[@]}"
+do
+    CLASSPATH="$i:$CLASSPATH"
+done
+
+#make it work for developers
+for d in "$ZOOBINDIR"/../build/lib/*.jar
+do
+   CLASSPATH="$d:$CLASSPATH"
+done
+
+#make it work for developers
+CLASSPATH="$ZOOBINDIR/../build/classes:$CLASSPATH"
+
+case "`uname`" in
+    CYGWIN*) cygwin=true ;;
+    *) cygwin=false ;;
+esac
+
+if $cygwin
+then
+    CLASSPATH=`cygpath -wp "$CLASSPATH"`
+fi
+
+#echo "CLASSPATH=$CLASSPATH"

bigdata-scripts_test3/roles/zookeeper/files/zkServer.sh

@@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# If this scripted is run out of /usr/bin or some other system bin directory
+# it should be linked to and not copied. Things like java jar files are found
+# relative to the canonical path of this script.
+#
+
+
+
+# use POSTIX interface, symlink is followed automatically
+ZOOBIN="${BASH_SOURCE-$0}"
+ZOOBIN="$(dirname "${ZOOBIN}")"
+ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)"
+
+if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then
+  . "$ZOOBINDIR/../libexec/zkEnv.sh"
+else
+  . "$ZOOBINDIR/zkEnv.sh"
+fi
+
+# See the following page for extensive details on setting
+# up the JVM to accept JMX remote management:
+# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+# by default we allow local JMX connections
+if [ "x$JMXLOCALONLY" = "x" ]
+then
+    JMXLOCALONLY=false
+fi
+
+if [ "x$JMXDISABLE" = "x" ] || [ "$JMXDISABLE" = 'false' ]
+then
+  echo "ZooKeeper JMX enabled by default" >&2
+  if [ "x$JMXPORT" = "x" ]
+  then
+    # for some reason these two options are necessary on jdk6 on Ubuntu
+    #   accord to the docs they are not necessary, but otw jconsole cannot
+    #   do a local attach
+    ZOOMAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY org.apache.zookeeper.server.quorum.QuorumPeerMain"
+  else
+    if [ "x$JMXAUTH" = "x" ]
+    then
+      JMXAUTH=false
+    fi
+    if [ "x$JMXSSL" = "x" ]
+    then
+      JMXSSL=false
+    fi
+    if [ "x$JMXLOG4J" = "x" ]
+    then
+      JMXLOG4J=true
+    fi
+    echo "ZooKeeper remote JMX Port set to $JMXPORT" >&2
+    echo "ZooKeeper remote JMX authenticate set to $JMXAUTH" >&2
+    echo "ZooKeeper remote JMX ssl set to $JMXSSL" >&2
+    echo "ZooKeeper remote JMX log4j set to $JMXLOG4J" >&2
+    ZOOMAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=$JMXPORT -Dcom.sun.management.jmxremote.authenticate=$JMXAUTH -Dcom.sun.management.jmxremote.ssl=$JMXSSL -Dzookeeper.jmx.log4j.disable=$JMXLOG4J org.apache.zookeeper.server.quorum.QuorumPeerMain"
+  fi
+else
+    echo "JMX disabled by user request" >&2
+    ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
+fi
+
+if [ "x$SERVER_JVMFLAGS"  != "x" ]
+then
+    JVMFLAGS="$SERVER_JVMFLAGS $JVMFLAGS"
+fi
+
+if [ "x$2" != "x" ]
+then
+    ZOOCFG="$ZOOCFGDIR/$2"
+fi
+
+# if we give a more complicated path to the config, don't screw around in $ZOOCFGDIR
+if [ "x$(dirname "$ZOOCFG")" != "x$ZOOCFGDIR" ]
+then
+    ZOOCFG="$2"
+fi
+
+if $cygwin
+then
+    ZOOCFG=`cygpath -wp "$ZOOCFG"`
+    # cygwin has a "kill" in the shell itself, gets confused
+    KILL=/bin/kill
+else
+    KILL=kill
+fi
+
+echo "Using config: $ZOOCFG" >&2
+
+case "$OSTYPE" in
+*solaris*)
+  GREP=/usr/xpg4/bin/grep
+  ;;
+*)
+  GREP=grep
+  ;;
+esac
+if [ -z "$ZOOPIDFILE" ]; then
+    ZOO_DATADIR="$($GREP "^[[:space:]]*dataDir" "$ZOOCFG" | sed -e 's/.*=//')"
+    if [ ! -d "$ZOO_DATADIR" ]; then
+        mkdir -p "$ZOO_DATADIR"
+    fi
+    ZOOPIDFILE="$ZOO_DATADIR/zookeeper_server.pid"
+else
+    # ensure it exists, otw stop will fail
+    mkdir -p "$(dirname "$ZOOPIDFILE")"
+fi
+
+if [ ! -w "$ZOO_LOG_DIR" ] ; then
+mkdir -p "$ZOO_LOG_DIR"
+fi
+
+_ZOO_DAEMON_OUT="$ZOO_LOG_DIR/zookeeper.log"
+
+case $1 in
+start)
+    echo  -n "Starting zookeeper ... "
+    if [ -f "$ZOOPIDFILE" ]; then
+      if kill -0 `cat "$ZOOPIDFILE"` > /dev/null 2>&1; then
+         echo $command already running as process `cat "$ZOOPIDFILE"`. 
+         exit 0
+      fi
+    fi
+    nohup "$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG" > "$_ZOO_DAEMON_OUT" 2>&1 < /dev/null &
+    if [ $? -eq 0 ]
+    then
+      case "$OSTYPE" in
+      *solaris*)
+        /bin/echo "${!}\\c" > "$ZOOPIDFILE"
+        ;;
+      *)
+        /bin/echo -n $! > "$ZOOPIDFILE"
+        ;;
+      esac
+      if [ $? -eq 0 ];
+      then
+        sleep 1
+        echo STARTED
+      else
+        echo FAILED TO WRITE PID
+        exit 1
+      fi
+    else
+      echo SERVER DID NOT START
+      exit 1
+    fi
+    ;;
+start-foreground)
+    ZOO_CMD=(exec "$JAVA")
+    if [ "${ZOO_NOEXEC}" != "" ]; then
+      ZOO_CMD=("$JAVA")
+    fi
+    "${ZOO_CMD[@]}" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    -cp "$CLASSPATH" $JVMFLAGS $ZOOMAIN "$ZOOCFG"
+    ;;
+print-cmd)
+    echo "\"$JAVA\" -Dzookeeper.log.dir=\"${ZOO_LOG_DIR}\" -Dzookeeper.root.logger=\"${ZOO_LOG4J_PROP}\" -cp \"$CLASSPATH\" $JVMFLAGS $ZOOMAIN \"$ZOOCFG\" > \"$_ZOO_DAEMON_OUT\" 2>&1 < /dev/null"
+    ;;
+stop)
+    echo -n "Stopping zookeeper ... "
+    if [ ! -f "$ZOOPIDFILE" ]
+    then
+      echo "no zookeeper to stop (could not find file $ZOOPIDFILE)"
+    else
+      $KILL -9 $(cat "$ZOOPIDFILE")
+      rm "$ZOOPIDFILE"
+      echo STOPPED
+    fi
+    exit 0
+    ;;
+upgrade)
+    shift
+    echo "upgrading the servers to 3.*"
+    "$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+    -cp "$CLASSPATH" $JVMFLAGS org.apache.zookeeper.server.upgrade.UpgradeMain ${@}
+    echo "Upgrading ... "
+    ;;
+restart)
+    shift
+    "$0" stop ${@}
+    sleep 3
+    "$0" start ${@}
+    ;;
+status)
+    # -q is necessary on some versions of linux where nc returns too quickly, and no stat result is output
+    clientPortAddress=`$GREP "^[[:space:]]*clientPortAddress[^[:alpha:]]" "$ZOOCFG" | sed -e 's/.*=//'`
+    if ! [ $clientPortAddress ]
+    then
+	clientPortAddress="localhost"
+    fi
+    clientPort=`$GREP "^[[:space:]]*clientPort[^[:alpha:]]" "$ZOOCFG" | sed -e 's/.*=//'`
+    STAT=`"$JAVA" "-Dzookeeper.log.dir=${ZOO_LOG_DIR}" "-Dzookeeper.root.logger=${ZOO_LOG4J_PROP}" \
+             -cp "$CLASSPATH" $JVMFLAGS org.apache.zookeeper.client.FourLetterWordMain \
+             $clientPortAddress $clientPort srvr 2> /dev/null    \
+          | $GREP Mode`
+    if [ "x$STAT" = "x" ]
+    then
+        echo "Error contacting service. It is probably not running."
+        exit 1
+    else
+        echo $STAT
+        exit 0
+    fi
+    ;;
+*)
+    echo "Usage: $0 {start|start-foreground|stop|restart|status|upgrade|print-cmd}" >&2
+
+esac

bigdata-scripts_test3/roles/zookeeper/files/zklogdelete.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+#只保留最近三天的日志，如需要多保留几天修改最后 -n days
+#将此脚本加载到系统定时任务中 /etc/crontab
+#脚本会读取环境变量，固需要配置环境变量。
+#. /etc/profile
+day=$(date  +"%Y-%m-%d" -d  "-3 days")
+
+zk=`jps | grep QuorumPeerMain | wc -l`
+if [[ $zk = "1" ]];then
+#echo $ZOOKEEPER_HOME/logs/system/*.$day*
+    rm -rf $ZOOKEEPER_HOME/logs/system/*.$day*
+fi
+
+kafka=`jps | grep Kafka | wc -l`
+if [[ $kafka = "1" ]];then
+#echo $KAFKA_HOME/logs/*.$day*
+    rm -rf $KAFKA_HOME/logs/*.$day*
+fi
+

bigdata-scripts_test3/roles/zookeeper/tasks/main.yml

@@ -0,0 +1,91 @@
+- name: "copy zookeeper install package to destination server"
+  copy:
+    src: "{{ role_path }}/files/{{ zookeeper.package_name }}"
+    dest: /tmp
+
+- name: "create zookeeper base_dir"
+  file:
+    path: '{{ zookeeper.base_dir }}'
+    state: touch
+
+- name: "install zookeeper"
+  unarchive:
+    src: "/tmp/{{ zookeeper.package_name }}"
+    dest: '{{ zookeeper.base_dir }}'
+    copy: no
+    mode: 0755
+
+- name: "copy any scripts"
+  copy:
+    src: '{{ item.src }}'
+    dest: '{{ item.dest }}'
+    mode: 0755
+  with_item:
+    - { src: '{{ role_path }}/files/dae-zookeeper.sh', dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/bin/' }
+    - { src: '{{ role_path }}/files/zkEnv.sh', dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/bin/' }
+    - { src: '{{ role_path }}/files/zklogdelete.sh', dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/bin/' }
+    - { src: '{{ role_path }}/files/zkServer.sh', dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/bin/' }
+
+- name: "copy log4j.properties"
+  copy:
+    src: "{{ role_path }}/files/log4j.properties"
+    dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/conf/'
+
+- name: "create zoo.cfg"
+  template:
+    src: "{{ role_path }}/templates/zoo.cfg.j2"
+    dest: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/conf/zoo.cfg'
+    mode: 0755
+
+- name: "create any dir"
+  file:
+    path: '{{ item.path }}'
+    state: '{{ item.state }}'
+  with_items:
+    - { path: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/data', state: directory }
+    - { path: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/logs', state: directory }
+    - { path: '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/data/myid', state: touch }
+
+- name: "echo 0 > restart_sum.log"
+  shell: echo 0 > '{{ zookeeper.base_dir }}/{{ zookeeper.version }}/logs/restart_sum.log'
+
+- name: "judge zoo.cfg id and ip"
+  shell: grep ":2888:3888" {{ zookeeper.base_dir }}/{{ zookeeper.version }}/conf/zoo.cfg
+  register: return
+  ignore_errors: true
+
+- name: "traverse ip/port"
+  shell: echo "server.{{ item.0 }}={{ item.1 }}:2888:3888" >> {{ zookeeper.base_dir }}/{{ zookeeper.version }}/conf/zoo.cfg
+  with_together:
+    - '{{ zookeeper.idlist }}'
+    - '{{ zookeeper.iplist }}'
+  when: return.rc != 0
+
+- name: "traverse zookeeper myid"
+  shell: echo "{{ zk_myid }}" > {{ zookeeper.base_dir }}/{{ zookeeper.version }}/data/myid
+ 
+- name: "template keepzkalive"
+  template:
+    src: "{{ role_path }}/templates/keepzkalive.j2"
+    dest: /etc/init.d/keepzkalive
+     
+- name: "start keepzkalive"
+  service:
+    name: keepzkalive
+    state: restarted
+    enabled: true
+
+- name: "template set_zk_profile.sh"
+  template:
+    src: "{{ role_path }}/templates/set_zk_profile.sh.j2"
+    dest: /tmp/set_zk_profile.sh
+    mode: 0755  
+
+- name: "judge zookeeper enviroment"
+  shell: grep "#zookeeper" /etc/profile
+  register: return
+  ignore_errors: true
+
+- name: "set_zk_profile"
+  shell: /bin/bash /tmp/set_zk_profile.sh
+  when: return.rc != 0

bigdata-scripts_test3/roles/zookeeper/templates/keepzkalive.j2

@@ -0,0 +1,11 @@
+#!/bin/bash
+# 
+# netconsole    This loads the netconsole module with the configured parameters.
+#
+# chkconfig:123456 20 80
+# description: keepzkalive
+source /etc/profile
+PRO_NAME=keepzklive
+
+killall -9 dae-zookeeper.sh
+{{ zookeeper.base_dir }}/{{ zookeeper.version }}/bin/dae-zookeeper.sh {{ zookeeper.base_dir }} > /dev/null 2>&1 &

bigdata-scripts_test3/roles/zookeeper/templates/set_zk_profile.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+echo -e ""##zookeeper"" >> /etc/profile
+echo -e "export ZOOKEEPER_HOME='{{ zookeeper.base_dir }}'/'{{ zookeeper.version }}'" >> /etc/profile
+echo -e "export PATH=\$ZOOKEEPER_HOME/bin:\$PATH" >> /etc/profile
+source /etc/profile

bigdata-scripts_test3/roles/zookeeper/templates/zoo.cfg.j2

@@ -0,0 +1,31 @@
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial 
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between 
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just 
+# example sakes.
+dataDir={{ zookeeper.base_dir }}/{{ version }}/data
+# the port at which the clients will connect
+clientPort=2181
+# the maximum number of client connections.
+# increase this if you need to handle more clients
+#maxClientCnxns=60
+#
+# Be sure to read the maintenance section of the 
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
+dataLogDir={{ zookeeper.base_dir }}/{{ zookeeper.version }}/log
+autopurge.purgeInterval=24
+

bigdata-scripts_test3/test_host/group_vars/all.yml

@@ -0,0 +1,24 @@
+zookeeper:
+    package_name: zookeeper-3.4.9.tar.gz
+    base_dir: /home/bigdata
+    version: zookeeper
+    idlist: [ '0','1','2' ]
+    iplist: [ '172.16.124.129','172.16.124.130','172.16.124.131' ]
+
+kafka:
+    ethname: ens33
+    package_name: kafka_2.11-1.0.0.tgz
+    base_dir: /home/bigdata
+    version: kafka_2.11-1.0.0
+    ethname: ens33
+    replica: 3
+    log_retentionBytes: 5368709120
+    log_retentionHours: 168
+    log_retentionCheckInterval_ms: 300000
+    mem_max: 1G
+    mem_min: 1G
+
+kafka_manager:
+    package_name: kafka-manager-1.3.3.18.zip
+    version: kafka-manager-1.3.3.18
+    

bigdata-scripts_test3/test_host/hosts

@@ -0,0 +1,13 @@
+[all:vars]
+ansible_user=root
+
+[server1]
+172.16.124.129 zk_myid=0 broker_id=0
+
+[server2]
+172.16.124.130 zk_myid=1 broker_id=1
+
+[server3]
+172.16.124.131 zk_myid=2 broker_id=2
+
+