首个ansible版本
This commit is contained in:
李玺康
@@ -1,22 +0,0 @@
|
|||||||
#为source channel sink起名
|
|
||||||
radius.channels = c1
|
|
||||||
radius.sinks = k1
|
|
||||||
#指定channel为kafka channel,省略source使得效率更高
|
|
||||||
radius.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
|
|
||||||
radius.channels.c1.kafka.bootstrap.servers = 192.168.40.152:9092
|
|
||||||
radius.channels.c1.kafka.topic = RADIUS-RECORD-LOG
|
|
||||||
radius.channels.c1.kafka.consumer.group.id = lxk_0514
|
|
||||||
radius.channels.c1.kafka.consumer.auto.offset.reset = latest
|
|
||||||
radius.channels.c1.kafka.consumer.enable.auto.commit = true
|
|
||||||
radius.channels.c1.kafka.consumer.fetch.max.wait.ms = 1000
|
|
||||||
radius.channels.c1.kafka.consumer.fetch.min.bytes = 1048576
|
|
||||||
radius.channels.c1.parseAsFlumeEvent = false
|
|
||||||
|
|
||||||
#原始file roll sink，将数据写到本机磁盘
|
|
||||||
radius.sinks.k1.type = file_roll
|
|
||||||
radius.sinks.k1.channel = c1
|
|
||||||
radius.sinks.k1.sink.pathManager = default
|
|
||||||
radius.sinks.k1.sink.pathManager.extension = dat
|
|
||||||
radius.sinks.k1.sink.pathManager.prefix = test-
|
|
||||||
radius.sinks.k1.sink.rollInterval = 30
|
|
||||||
radius.sinks.k1.sink.directory = /home/192.168.60.101/RADIUS-RECORD-LOG
|
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
#为source channel sink起名
|
||||||
|
active_denfence.channels = c1
|
||||||
|
active_denfence.sinks = k1
|
||||||
|
#指定channel为kafka channel,省略source使得效率更高
|
||||||
|
active_denfence.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
active_denfence.channels.c1.kafka.bootstrap.servers = 192.168.40.152:9092
|
||||||
|
active_denfence.channels.c1.kafka.topic = ACTIVE-DEFENCE-EVENT-LOG
|
||||||
|
active_denfence.channels.c1.kafka.consumer.group.id = lxk_0514
|
||||||
|
active_denfence.channels.c1.kafka.consumer.auto.offset.reset = latest
|
||||||
|
active_denfence.channels.c1.kafka.consumer.enable.auto.commit = true
|
||||||
|
active_denfence.channels.c1.kafka.consumer.fetch.max.wait.ms = 100
|
||||||
|
active_denfence.channels.c1.kafka.consumer.fetch.min.bytes = 1048576
|
||||||
|
active_denfence.channels.c1.parseAsFlumeEvent = false
|
||||||
|
|
||||||
|
#原始file roll sink，将数据写到本机磁盘
|
||||||
|
active_denfence.sinks.k1.type = file_roll
|
||||||
|
active_denfence.sinks.k1.channel = c1
|
||||||
|
active_denfence.sinks.k1.sink.pathManager = default
|
||||||
|
active_denfence.sinks.k1.sink.pathManager.extension = dat
|
||||||
|
active_denfence.sinks.k1.sink.pathManager.prefix = test-
|
||||||
|
active_denfence.sinks.k1.sink.rollInterval = 30
|
||||||
|
active_denfence.sinks.k1.sink.directory = /home/192.168.60.101/ACTIVE-DEFENCE-EVENT-LOG
|
||||||
@@ -1,15 +1,15 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
|
|
||||||
#JAR_NAME=ktk_ip_asn.properties
|
#JAR_NAME=ktk_ip_asn.properties
|
||||||
PROPERTIES_NAME=security_f2k
|
PROPERTIES_NAME=active_denfence_k2f
|
||||||
#flume进程名称
|
#flume进程名称
|
||||||
FLUME_NAME=security
|
FLUME_NAME=active_denfence
|
||||||
#flume根目录
|
#flume根目录
|
||||||
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
||||||
#当前路径
|
#当前路径
|
||||||
CONF_DIR=$(cd $(dirname $0); pwd)
|
CONF_DIR=$(cd $(dirname $0); pwd)
|
||||||
#进程总数
|
#进程总数
|
||||||
PROCESS_SUM=3
|
PROCESS_SUM=5
|
||||||
|
|
||||||
echo "##############################################################" >> ${CONF_DIR}/restart_log/restart_${PROPERTIES_NAME}.log
|
echo "##############################################################" >> ${CONF_DIR}/restart_log/restart_${PROPERTIES_NAME}.log
|
||||||
id=0 #配置文件后面的序号，无特殊作用
|
id=0 #配置文件后面的序号，无特殊作用
|
||||||
@@ -55,3 +55,42 @@
|
|||||||
20200513102434 ---> the connection_f2k_APP restart ---> 3
|
20200513102434 ---> the connection_f2k_APP restart ---> 3
|
||||||
20200513102444 ---> the connection_f2k_APP restart ---> 4
|
20200513102444 ---> the connection_f2k_APP restart ---> 4
|
||||||
20200513102454 ---> the connection_f2k_APP restart ---> 5
|
20200513102454 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
20200513104430 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200513104440 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200513104450 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200513104500 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200513104510 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
##############################################################
|
||||||
|
20200513181554 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200513181604 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200513181614 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200513181624 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200513181634 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
##############################################################
|
||||||
|
20200513182119 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200513182129 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200513182139 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200513182149 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200513182159 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
##############################################################
|
||||||
|
20200513182640 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200513182651 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200513182701 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200513182711 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200513182721 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
##############################################################
|
||||||
|
20200514171459 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200514171509 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200514171519 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200514171529 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200514171539 ---> the connection_f2k_APP restart ---> 5
|
||||||
|
##############################################################
|
||||||
|
20200514172152 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200514172202 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200514172212 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
##############################################################
|
||||||
|
20200514172234 ---> the connection_f2k_APP restart ---> 1
|
||||||
|
20200514172244 ---> the connection_f2k_APP restart ---> 2
|
||||||
|
20200514172254 ---> the connection_f2k_APP restart ---> 3
|
||||||
|
20200514172304 ---> the connection_f2k_APP restart ---> 4
|
||||||
|
20200514172314 ---> the connection_f2k_APP restart ---> 5
|
||||||
201
external-flume/conf_external/connection/logs/flume.log
Normal file
201
external-flume/conf_external/connection/logs/flume.log
Normal file
@@ -0,0 +1,201 @@
|
|||||||
|
14 May 2020 11:46:42,409 INFO [lifecycleSupervisor-1-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider.start:62) - Configuration provider starting
|
||||||
|
14 May 2020 11:46:42,421 INFO [conf-file-poller-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run:138) - Reloading configuration file:/home/apache-flume-1.9.0-bin/conf/connection/connection_k2f.properties
|
||||||
|
14 May 2020 11:46:42,431 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,433 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,433 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,434 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,434 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addProperty:1117) - Added sinks: k1 Agent: connection
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,437 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,437 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,438 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,440 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,440 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,441 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,441 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateConfigFilterSet:623) - Agent configuration for 'connection' has no configfilters.
|
||||||
|
14 May 2020 11:46:42,464 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSources:702) - Agent configuration for 'connection' has no sources.
|
||||||
|
14 May 2020 11:46:42,469 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration.validateConfiguration:163) - Post-validation flume configuration contains configuration for agents: [connection]
|
||||||
|
14 May 2020 11:46:42,469 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:151) - Creating channels
|
||||||
|
14 May 2020 11:46:42,481 INFO [conf-file-poller-0] (org.apache.flume.channel.DefaultChannelFactory.create:42) - Creating instance of channel c1 type org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
14 May 2020 11:46:42,503 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:205) - Created channel c1
|
||||||
|
14 May 2020 11:46:42,504 INFO [conf-file-poller-0] (org.apache.flume.sink.DefaultSinkFactory.create:42) - Creating instance of sink: k1, type: file_roll
|
||||||
|
14 May 2020 11:46:42,514 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.getConfiguration:120) - Channel c1 connected to [k1]
|
||||||
|
14 May 2020 11:46:42,524 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:162) - Starting new configuration:{ sourceRunners:{} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@5cf2aeb7 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.kafka.KafkaChannel{name: c1}} }
|
||||||
|
14 May 2020 11:46:42,525 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:169) - Starting Channel c1
|
||||||
|
14 May 2020 11:46:42,526 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:135) - Starting Kafka Channel: c1
|
||||||
|
14 May 2020 11:46:42,562 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ProducerConfig values:
|
||||||
|
acks = all
|
||||||
|
batch.size = 16384
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
buffer.memory = 33554432
|
||||||
|
client.id =
|
||||||
|
compression.type = none
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
enable.idempotence = false
|
||||||
|
interceptor.classes = []
|
||||||
|
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
|
||||||
|
linger.ms = 0
|
||||||
|
max.block.ms = 60000
|
||||||
|
max.in.flight.requests.per.connection = 5
|
||||||
|
max.request.size = 1048576
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
|
||||||
|
receive.buffer.bytes = 32768
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retries = 0
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
transaction.timeout.ms = 60000
|
||||||
|
transactional.id = null
|
||||||
|
value.serializer = class org.apache.kafka.common.serialization.ByteArraySerializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,638 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,639 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,641 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:143) - Topic = CONNECTION-RECORD-LOG
|
||||||
|
14 May 2020 11:46:42,641 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,642 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: CHANNEL, name: c1 started
|
||||||
|
14 May 2020 11:46:42,642 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:196) - Starting Sink k1
|
||||||
|
14 May 2020 11:46:42,643 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:111) - Starting org.apache.flume.sink.RollingFileSink{name:k1, channel:c1}...
|
||||||
|
14 May 2020 11:46:42,644 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,644 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: SINK, name: k1 started
|
||||||
|
14 May 2020 11:46:42,645 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:143) - RollingFileSink k1 started.
|
||||||
|
14 May 2020 11:46:42,661 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ConsumerConfig values:
|
||||||
|
auto.commit.interval.ms = 5000
|
||||||
|
auto.offset.reset = latest
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
check.crcs = true
|
||||||
|
client.id =
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
default.api.timeout.ms = 60000
|
||||||
|
enable.auto.commit = false
|
||||||
|
exclude.internal.topics = true
|
||||||
|
fetch.max.bytes = 52428800
|
||||||
|
fetch.max.wait.ms = 1000
|
||||||
|
fetch.min.bytes = 1048576
|
||||||
|
group.id = lxk_0514
|
||||||
|
heartbeat.interval.ms = 3000
|
||||||
|
interceptor.classes = []
|
||||||
|
internal.leave.group.on.close = true
|
||||||
|
isolation.level = read_uncommitted
|
||||||
|
key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
|
||||||
|
max.partition.fetch.bytes = 1048576
|
||||||
|
max.poll.interval.ms = 300000
|
||||||
|
max.poll.records = 500
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
|
||||||
|
receive.buffer.bytes = 65536
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
session.timeout.ms = 10000
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
value.deserializer = class org.apache.kafka.common.serialization.ByteArrayDeserializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,696 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,697 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,700 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.KafkaChannel.createConsumerAndRecords:304) - Created new consumer to connect to Kafka
|
||||||
|
14 May 2020 11:46:42,870 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 11:46:42,871 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$FindCoordinatorResponseHandler.onSuccess:677) - [Consumer clientId=consumer-1, groupId=lxk_0514] Discovered group coordinator 192.168.40.152:9092 (id: 2147483646 rack: null)
|
||||||
|
14 May 2020 11:46:42,873 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions []
|
||||||
|
14 May 2020 11:46:42,873 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:42,887 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 18
|
||||||
|
14 May 2020 11:46:42,888 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:42,889 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:46:45,901 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 11:46:45,910 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:45,910 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 11:46:45,911 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:45,921 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 19
|
||||||
|
14 May 2020 11:46:45,922 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:45,923 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:51:42,641 INFO [kafka-producer-network-thread | producer-1] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 12:02:48,119 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:48,715 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:48,716 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 12:02:48,717 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:54,224 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 20
|
||||||
|
14 May 2020 12:02:54,225 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:54,225 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 12:02:57,326 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:59,720 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:59,721 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 12:02:59,722 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:59,729 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 21
|
||||||
|
14 May 2020 12:02:59,730 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:59,730 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
@@ -66,3 +66,7 @@
|
|||||||
20200513192014 ---> the connection_k2f_APP restart ---> 1
|
20200513192014 ---> the connection_k2f_APP restart ---> 1
|
||||||
##############################################################
|
##############################################################
|
||||||
20200513192212 ---> the connection_k2f_APP restart ---> 1
|
20200513192212 ---> the connection_k2f_APP restart ---> 1
|
||||||
|
##############################################################
|
||||||
|
20200514112452 ---> the connection_k2f_APP restart ---> 1
|
||||||
|
##############################################################
|
||||||
|
20200514114641 ---> the connection_k2f_APP restart ---> 1
|
||||||
359
external-flume/conf_external/proxy/logs/flume.log
Normal file
359
external-flume/conf_external/proxy/logs/flume.log
Normal file
@@ -0,0 +1,359 @@
|
|||||||
|
14 May 2020 11:46:42,712 INFO [lifecycleSupervisor-1-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider.start:62) - Configuration provider starting
|
||||||
|
14 May 2020 11:46:42,720 INFO [conf-file-poller-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run:138) - Reloading configuration file:/home/apache-flume-1.9.0-bin/conf/proxy/proxy_k2f.properties
|
||||||
|
14 May 2020 11:46:42,725 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,725 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,726 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,726 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,726 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,727 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,727 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,727 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,728 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,728 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,728 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addProperty:1117) - Added sinks: k1 Agent: proxy
|
||||||
|
14 May 2020 11:46:42,728 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,728 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,729 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,730 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,730 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,730 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,731 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateConfigFilterSet:623) - Agent configuration for 'proxy' has no configfilters.
|
||||||
|
14 May 2020 11:46:42,741 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSources:702) - Agent configuration for 'proxy' has no sources.
|
||||||
|
14 May 2020 11:46:42,743 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration.validateConfiguration:163) - Post-validation flume configuration contains configuration for agents: [proxy]
|
||||||
|
14 May 2020 11:46:42,743 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:151) - Creating channels
|
||||||
|
14 May 2020 11:46:42,749 INFO [conf-file-poller-0] (org.apache.flume.channel.DefaultChannelFactory.create:42) - Creating instance of channel c1 type org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
14 May 2020 11:46:42,761 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:205) - Created channel c1
|
||||||
|
14 May 2020 11:46:42,761 INFO [conf-file-poller-0] (org.apache.flume.sink.DefaultSinkFactory.create:42) - Creating instance of sink: k1, type: file_roll
|
||||||
|
14 May 2020 11:46:42,767 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.getConfiguration:120) - Channel c1 connected to [k1]
|
||||||
|
14 May 2020 11:46:42,774 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:162) - Starting new configuration:{ sourceRunners:{} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@7e8b5303 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.kafka.KafkaChannel{name: c1}} }
|
||||||
|
14 May 2020 11:46:42,774 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:169) - Starting Channel c1
|
||||||
|
14 May 2020 11:46:42,775 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:135) - Starting Kafka Channel: c1
|
||||||
|
14 May 2020 11:46:42,799 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ProducerConfig values:
|
||||||
|
acks = all
|
||||||
|
batch.size = 16384
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
buffer.memory = 33554432
|
||||||
|
client.id =
|
||||||
|
compression.type = none
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
enable.idempotence = false
|
||||||
|
interceptor.classes = []
|
||||||
|
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
|
||||||
|
linger.ms = 0
|
||||||
|
max.block.ms = 60000
|
||||||
|
max.in.flight.requests.per.connection = 5
|
||||||
|
max.request.size = 1048576
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
|
||||||
|
receive.buffer.bytes = 32768
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retries = 0
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
transaction.timeout.ms = 60000
|
||||||
|
transactional.id = null
|
||||||
|
value.serializer = class org.apache.kafka.common.serialization.ByteArraySerializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,854 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,855 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,857 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:143) - Topic = PROXY-EVENT-LOG
|
||||||
|
14 May 2020 11:46:42,857 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,858 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: CHANNEL, name: c1 started
|
||||||
|
14 May 2020 11:46:42,858 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:196) - Starting Sink k1
|
||||||
|
14 May 2020 11:46:42,859 INFO [lifecycleSupervisor-1-3] (org.apache.flume.sink.RollingFileSink.start:111) - Starting org.apache.flume.sink.RollingFileSink{name:k1, channel:c1}...
|
||||||
|
14 May 2020 11:46:42,860 INFO [lifecycleSupervisor-1-3] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,860 INFO [lifecycleSupervisor-1-3] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: SINK, name: k1 started
|
||||||
|
14 May 2020 11:46:42,862 INFO [lifecycleSupervisor-1-3] (org.apache.flume.sink.RollingFileSink.start:143) - RollingFileSink k1 started.
|
||||||
|
14 May 2020 11:46:42,884 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ConsumerConfig values:
|
||||||
|
auto.commit.interval.ms = 5000
|
||||||
|
auto.offset.reset = latest
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
check.crcs = true
|
||||||
|
client.id =
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
default.api.timeout.ms = 60000
|
||||||
|
enable.auto.commit = false
|
||||||
|
exclude.internal.topics = true
|
||||||
|
fetch.max.bytes = 52428800
|
||||||
|
fetch.max.wait.ms = 1000
|
||||||
|
fetch.min.bytes = 1048576
|
||||||
|
group.id = lxk_0514
|
||||||
|
heartbeat.interval.ms = 3000
|
||||||
|
interceptor.classes = []
|
||||||
|
internal.leave.group.on.close = true
|
||||||
|
isolation.level = read_uncommitted
|
||||||
|
key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
|
||||||
|
max.partition.fetch.bytes = 1048576
|
||||||
|
max.poll.interval.ms = 300000
|
||||||
|
max.poll.records = 500
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
|
||||||
|
receive.buffer.bytes = 65536
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
session.timeout.ms = 10000
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
value.deserializer = class org.apache.kafka.common.serialization.ByteArrayDeserializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,934 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,934 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,939 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.KafkaChannel.createConsumerAndRecords:304) - Created new consumer to connect to Kafka
|
||||||
|
14 May 2020 11:46:43,146 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 11:46:43,147 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$FindCoordinatorResponseHandler.onSuccess:677) - [Consumer clientId=consumer-1, groupId=lxk_0514] Discovered group coordinator 192.168.40.152:9092 (id: 2147483646 rack: null)
|
||||||
|
14 May 2020 11:46:43,149 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions []
|
||||||
|
14 May 2020 11:46:43,149 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:46,953 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 19
|
||||||
|
14 May 2020 11:46:46,956 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [PROXY-EVENT-LOG-0]
|
||||||
|
14 May 2020 11:46:46,957 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic PROXY-EVENT-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:51:42,863 INFO [kafka-producer-network-thread | producer-1] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 12:02:55,605 INFO [lifecycleSupervisor-1-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider.start:62) - Configuration provider starting
|
||||||
|
14 May 2020 12:02:55,618 INFO [conf-file-poller-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run:138) - Reloading configuration file:/home/apache-flume-1.9.0-bin/conf/proxy/proxy_k2f.properties
|
||||||
|
14 May 2020 12:02:55,622 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,623 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,624 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,625 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,625 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,625 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addProperty:1117) - Added sinks: k1 Agent: proxy
|
||||||
|
14 May 2020 12:02:55,625 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,626 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 12:02:55,626 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,627 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,627 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,628 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 12:02:55,628 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateConfigFilterSet:623) - Agent configuration for 'proxy' has no configfilters.
|
||||||
|
14 May 2020 12:02:55,638 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSources:702) - Agent configuration for 'proxy' has no sources.
|
||||||
|
14 May 2020 12:02:55,641 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration.validateConfiguration:163) - Post-validation flume configuration contains configuration for agents: [proxy]
|
||||||
|
14 May 2020 12:02:55,641 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:151) - Creating channels
|
||||||
|
14 May 2020 12:02:55,647 INFO [conf-file-poller-0] (org.apache.flume.channel.DefaultChannelFactory.create:42) - Creating instance of channel c1 type org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
14 May 2020 12:02:55,659 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:205) - Created channel c1
|
||||||
|
14 May 2020 12:02:55,660 INFO [conf-file-poller-0] (org.apache.flume.sink.DefaultSinkFactory.create:42) - Creating instance of sink: k1, type: file_roll
|
||||||
|
14 May 2020 12:02:55,666 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.getConfiguration:120) - Channel c1 connected to [k1]
|
||||||
|
14 May 2020 12:02:55,672 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:162) - Starting new configuration:{ sourceRunners:{} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@aba23f8 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.kafka.KafkaChannel{name: c1}} }
|
||||||
|
14 May 2020 12:02:55,673 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:169) - Starting Channel c1
|
||||||
|
14 May 2020 12:02:55,674 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:135) - Starting Kafka Channel: c1
|
||||||
|
14 May 2020 12:02:55,713 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ProducerConfig values:
|
||||||
|
acks = all
|
||||||
|
batch.size = 16384
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
buffer.memory = 33554432
|
||||||
|
client.id =
|
||||||
|
compression.type = none
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
enable.idempotence = false
|
||||||
|
interceptor.classes = []
|
||||||
|
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
|
||||||
|
linger.ms = 0
|
||||||
|
max.block.ms = 60000
|
||||||
|
max.in.flight.requests.per.connection = 5
|
||||||
|
max.request.size = 1048576
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
|
||||||
|
receive.buffer.bytes = 32768
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retries = 0
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
transaction.timeout.ms = 60000
|
||||||
|
transactional.id = null
|
||||||
|
value.serializer = class org.apache.kafka.common.serialization.ByteArraySerializer
|
||||||
|
|
||||||
|
14 May 2020 12:02:55,821 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 12:02:55,821 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 12:02:55,824 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:143) - Topic = PROXY-EVENT-LOG
|
||||||
|
14 May 2020 12:02:55,825 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
|
||||||
|
14 May 2020 12:02:55,826 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: CHANNEL, name: c1 started
|
||||||
|
14 May 2020 12:02:55,826 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:196) - Starting Sink k1
|
||||||
|
14 May 2020 12:02:55,827 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:111) - Starting org.apache.flume.sink.RollingFileSink{name:k1, channel:c1}...
|
||||||
|
14 May 2020 12:02:55,829 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
|
||||||
|
14 May 2020 12:02:55,829 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: SINK, name: k1 started
|
||||||
|
14 May 2020 12:02:55,831 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:143) - RollingFileSink k1 started.
|
||||||
|
14 May 2020 12:02:55,858 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ConsumerConfig values:
|
||||||
|
auto.commit.interval.ms = 5000
|
||||||
|
auto.offset.reset = latest
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
check.crcs = true
|
||||||
|
client.id =
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
default.api.timeout.ms = 60000
|
||||||
|
enable.auto.commit = false
|
||||||
|
exclude.internal.topics = true
|
||||||
|
fetch.max.bytes = 52428800
|
||||||
|
fetch.max.wait.ms = 1000
|
||||||
|
fetch.min.bytes = 1048576
|
||||||
|
group.id = lxk_0514
|
||||||
|
heartbeat.interval.ms = 3000
|
||||||
|
interceptor.classes = []
|
||||||
|
internal.leave.group.on.close = true
|
||||||
|
isolation.level = read_uncommitted
|
||||||
|
key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
|
||||||
|
max.partition.fetch.bytes = 1048576
|
||||||
|
max.poll.interval.ms = 300000
|
||||||
|
max.poll.records = 500
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
|
||||||
|
receive.buffer.bytes = 65536
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
session.timeout.ms = 10000
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
value.deserializer = class org.apache.kafka.common.serialization.ByteArrayDeserializer
|
||||||
|
|
||||||
|
14 May 2020 12:02:55,909 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 12:02:55,909 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 12:02:55,912 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.KafkaChannel.createConsumerAndRecords:304) - Created new consumer to connect to Kafka
|
||||||
|
14 May 2020 12:02:56,156 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 12:02:56,244 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$FindCoordinatorResponseHandler.onSuccess:677) - [Consumer clientId=consumer-1, groupId=lxk_0514] Discovered group coordinator 192.168.40.152:9092 (id: 2147483646 rack: null)
|
||||||
|
14 May 2020 12:02:56,248 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions []
|
||||||
|
14 May 2020 12:02:56,249 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:59,926 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 21
|
||||||
|
14 May 2020 12:02:59,928 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [PROXY-EVENT-LOG-0]
|
||||||
|
14 May 2020 12:02:59,928 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic PROXY-EVENT-LOG - partition 0 assigned.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
##############################################################
|
||||||
|
20200514112452 ---> the proxy_k2f_APP restart ---> 1
|
||||||
|
##############################################################
|
||||||
|
20200514114642 ---> the proxy_k2f_APP restart ---> 1
|
||||||
|
20200514120255 ---> the proxy_k2f_APP restart ---> 1
|
||||||
189
external-flume/conf_external/security/logs/flume.log
Normal file
189
external-flume/conf_external/security/logs/flume.log
Normal file
@@ -0,0 +1,189 @@
|
|||||||
|
14 May 2020 11:46:42,542 INFO [lifecycleSupervisor-1-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider.start:62) - Configuration provider starting
|
||||||
|
14 May 2020 11:46:42,551 INFO [conf-file-poller-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run:138) - Reloading configuration file:/home/apache-flume-1.9.0-bin/conf/security/security_k2f.properties
|
||||||
|
14 May 2020 11:46:42,556 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,557 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addProperty:1117) - Added sinks: k1 Agent: security
|
||||||
|
14 May 2020 11:46:42,557 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,557 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,557 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,558 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,559 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,559 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,559 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,560 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,561 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,561 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,561 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateConfigFilterSet:623) - Agent configuration for 'security' has no configfilters.
|
||||||
|
14 May 2020 11:46:42,571 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSources:702) - Agent configuration for 'security' has no sources.
|
||||||
|
14 May 2020 11:46:42,573 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration.validateConfiguration:163) - Post-validation flume configuration contains configuration for agents: [security]
|
||||||
|
14 May 2020 11:46:42,574 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:151) - Creating channels
|
||||||
|
14 May 2020 11:46:42,579 INFO [conf-file-poller-0] (org.apache.flume.channel.DefaultChannelFactory.create:42) - Creating instance of channel c1 type org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
14 May 2020 11:46:42,591 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:205) - Created channel c1
|
||||||
|
14 May 2020 11:46:42,592 INFO [conf-file-poller-0] (org.apache.flume.sink.DefaultSinkFactory.create:42) - Creating instance of sink: k1, type: file_roll
|
||||||
|
14 May 2020 11:46:42,598 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.getConfiguration:120) - Channel c1 connected to [k1]
|
||||||
|
14 May 2020 11:46:42,604 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:162) - Starting new configuration:{ sourceRunners:{} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@7e8b5303 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.kafka.KafkaChannel{name: c1}} }
|
||||||
|
14 May 2020 11:46:42,605 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:169) - Starting Channel c1
|
||||||
|
14 May 2020 11:46:42,605 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:135) - Starting Kafka Channel: c1
|
||||||
|
14 May 2020 11:46:42,636 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ProducerConfig values:
|
||||||
|
acks = all
|
||||||
|
batch.size = 16384
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
buffer.memory = 33554432
|
||||||
|
client.id =
|
||||||
|
compression.type = none
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
enable.idempotence = false
|
||||||
|
interceptor.classes = []
|
||||||
|
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
|
||||||
|
linger.ms = 0
|
||||||
|
max.block.ms = 60000
|
||||||
|
max.in.flight.requests.per.connection = 5
|
||||||
|
max.request.size = 1048576
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
|
||||||
|
receive.buffer.bytes = 32768
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retries = 0
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
transaction.timeout.ms = 60000
|
||||||
|
transactional.id = null
|
||||||
|
value.serializer = class org.apache.kafka.common.serialization.ByteArraySerializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,692 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,692 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,694 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:143) - Topic = SECURITY-EVENT-LOG
|
||||||
|
14 May 2020 11:46:42,695 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,695 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: CHANNEL, name: c1 started
|
||||||
|
14 May 2020 11:46:42,695 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:196) - Starting Sink k1
|
||||||
|
14 May 2020 11:46:42,697 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:111) - Starting org.apache.flume.sink.RollingFileSink{name:k1, channel:c1}...
|
||||||
|
14 May 2020 11:46:42,698 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,698 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: SINK, name: k1 started
|
||||||
|
14 May 2020 11:46:42,700 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:143) - RollingFileSink k1 started.
|
||||||
|
14 May 2020 11:46:42,734 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ConsumerConfig values:
|
||||||
|
auto.commit.interval.ms = 5000
|
||||||
|
auto.offset.reset = latest
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
check.crcs = true
|
||||||
|
client.id =
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
default.api.timeout.ms = 60000
|
||||||
|
enable.auto.commit = false
|
||||||
|
exclude.internal.topics = true
|
||||||
|
fetch.max.bytes = 52428800
|
||||||
|
fetch.max.wait.ms = 1000
|
||||||
|
fetch.min.bytes = 1048576
|
||||||
|
group.id = lxk_0514
|
||||||
|
heartbeat.interval.ms = 3000
|
||||||
|
interceptor.classes = []
|
||||||
|
internal.leave.group.on.close = true
|
||||||
|
isolation.level = read_uncommitted
|
||||||
|
key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
|
||||||
|
max.partition.fetch.bytes = 1048576
|
||||||
|
max.poll.interval.ms = 300000
|
||||||
|
max.poll.records = 500
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
|
||||||
|
receive.buffer.bytes = 65536
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
session.timeout.ms = 10000
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
value.deserializer = class org.apache.kafka.common.serialization.ByteArrayDeserializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,796 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,796 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,800 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.KafkaChannel.createConsumerAndRecords:304) - Created new consumer to connect to Kafka
|
||||||
|
14 May 2020 11:46:43,000 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 11:46:43,001 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$FindCoordinatorResponseHandler.onSuccess:677) - [Consumer clientId=consumer-1, groupId=lxk_0514] Discovered group coordinator 192.168.40.152:9092 (id: 2147483646 rack: null)
|
||||||
|
14 May 2020 11:46:43,003 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions []
|
||||||
|
14 May 2020 11:46:43,003 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:46,812 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 19
|
||||||
|
14 May 2020 11:46:46,814 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [SECURITY-EVENT-LOG-0]
|
||||||
|
14 May 2020 11:46:46,815 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic SECURITY-EVENT-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:51:42,694 INFO [kafka-producer-network-thread | producer-1] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 12:02:49,163 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:52,169 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:52,762 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [SECURITY-EVENT-LOG-0]
|
||||||
|
14 May 2020 12:02:52,763 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic SECURITY-EVENT-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 12:02:52,763 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:58,267 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:03:03,769 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 21
|
||||||
|
14 May 2020 12:03:03,770 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [SECURITY-EVENT-LOG-0]
|
||||||
|
14 May 2020 12:03:03,771 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic SECURITY-EVENT-LOG - partition 0 assigned.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
##############################################################
|
||||||
|
20200514112452 ---> the security_k2f_APP restart ---> 1
|
||||||
|
##############################################################
|
||||||
|
20200514114642 ---> the security_k2f_APP restart ---> 1
|
||||||
@@ -1,9 +1,9 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
|
|
||||||
#JAR_NAME=ktk_ip_asn.properties
|
#JAR_NAME=ktk_ip_asn.properties
|
||||||
PROPERTIES_NAME=radius_k2f
|
PROPERTIES_NAME=connection_k2f
|
||||||
#flume进程名称
|
#flume进程名称
|
||||||
FLUME_NAME=radius
|
FLUME_NAME=connection
|
||||||
#flume根目录
|
#flume根目录
|
||||||
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
BASE_DIR=$(cd $(dirname $0); cd ../../; pwd)
|
||||||
#当前路径
|
#当前路径
|
||||||
201
external-flume/conf_external/traffic/logs/flume.log
Normal file
201
external-flume/conf_external/traffic/logs/flume.log
Normal file
@@ -0,0 +1,201 @@
|
|||||||
|
14 May 2020 11:46:42,409 INFO [lifecycleSupervisor-1-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider.start:62) - Configuration provider starting
|
||||||
|
14 May 2020 11:46:42,421 INFO [conf-file-poller-0] (org.apache.flume.node.PollingPropertiesFileConfigurationProvider$FileWatcherRunnable.run:138) - Reloading configuration file:/home/apache-flume-1.9.0-bin/conf/connection/connection_k2f.properties
|
||||||
|
14 May 2020 11:46:42,431 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,433 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,433 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,434 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,434 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addProperty:1117) - Added sinks: k1 Agent: connection
|
||||||
|
14 May 2020 11:46:42,435 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,436 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,437 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,437 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:k1
|
||||||
|
14 May 2020 11:46:42,438 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,440 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,440 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,441 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.addComponentConfig:1203) - Processing:c1
|
||||||
|
14 May 2020 11:46:42,441 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateConfigFilterSet:623) - Agent configuration for 'connection' has no configfilters.
|
||||||
|
14 May 2020 11:46:42,464 WARN [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration$AgentConfiguration.validateSources:702) - Agent configuration for 'connection' has no sources.
|
||||||
|
14 May 2020 11:46:42,469 INFO [conf-file-poller-0] (org.apache.flume.conf.FlumeConfiguration.validateConfiguration:163) - Post-validation flume configuration contains configuration for agents: [connection]
|
||||||
|
14 May 2020 11:46:42,469 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:151) - Creating channels
|
||||||
|
14 May 2020 11:46:42,481 INFO [conf-file-poller-0] (org.apache.flume.channel.DefaultChannelFactory.create:42) - Creating instance of channel c1 type org.apache.flume.channel.kafka.KafkaChannel
|
||||||
|
14 May 2020 11:46:42,503 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.loadChannels:205) - Created channel c1
|
||||||
|
14 May 2020 11:46:42,504 INFO [conf-file-poller-0] (org.apache.flume.sink.DefaultSinkFactory.create:42) - Creating instance of sink: k1, type: file_roll
|
||||||
|
14 May 2020 11:46:42,514 INFO [conf-file-poller-0] (org.apache.flume.node.AbstractConfigurationProvider.getConfiguration:120) - Channel c1 connected to [k1]
|
||||||
|
14 May 2020 11:46:42,524 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:162) - Starting new configuration:{ sourceRunners:{} sinkRunners:{k1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@5cf2aeb7 counterGroup:{ name:null counters:{} } }} channels:{c1=org.apache.flume.channel.kafka.KafkaChannel{name: c1}} }
|
||||||
|
14 May 2020 11:46:42,525 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:169) - Starting Channel c1
|
||||||
|
14 May 2020 11:46:42,526 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:135) - Starting Kafka Channel: c1
|
||||||
|
14 May 2020 11:46:42,562 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ProducerConfig values:
|
||||||
|
acks = all
|
||||||
|
batch.size = 16384
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
buffer.memory = 33554432
|
||||||
|
client.id =
|
||||||
|
compression.type = none
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
enable.idempotence = false
|
||||||
|
interceptor.classes = []
|
||||||
|
key.serializer = class org.apache.kafka.common.serialization.StringSerializer
|
||||||
|
linger.ms = 0
|
||||||
|
max.block.ms = 60000
|
||||||
|
max.in.flight.requests.per.connection = 5
|
||||||
|
max.request.size = 1048576
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partitioner.class = class org.apache.kafka.clients.producer.internals.DefaultPartitioner
|
||||||
|
receive.buffer.bytes = 32768
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retries = 0
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
transaction.timeout.ms = 60000
|
||||||
|
transactional.id = null
|
||||||
|
value.serializer = class org.apache.kafka.common.serialization.ByteArraySerializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,638 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,639 INFO [lifecycleSupervisor-1-0] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,641 INFO [lifecycleSupervisor-1-0] (org.apache.flume.channel.kafka.KafkaChannel.start:143) - Topic = CONNECTION-RECORD-LOG
|
||||||
|
14 May 2020 11:46:42,641 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: CHANNEL, name: c1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,642 INFO [lifecycleSupervisor-1-0] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: CHANNEL, name: c1 started
|
||||||
|
14 May 2020 11:46:42,642 INFO [conf-file-poller-0] (org.apache.flume.node.Application.startAllComponents:196) - Starting Sink k1
|
||||||
|
14 May 2020 11:46:42,643 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:111) - Starting org.apache.flume.sink.RollingFileSink{name:k1, channel:c1}...
|
||||||
|
14 May 2020 11:46:42,644 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.register:119) - Monitored counter group for type: SINK, name: k1: Successfully registered new MBean.
|
||||||
|
14 May 2020 11:46:42,644 INFO [lifecycleSupervisor-1-1] (org.apache.flume.instrumentation.MonitoredCounterGroup.start:95) - Component type: SINK, name: k1 started
|
||||||
|
14 May 2020 11:46:42,645 INFO [lifecycleSupervisor-1-1] (org.apache.flume.sink.RollingFileSink.start:143) - RollingFileSink k1 started.
|
||||||
|
14 May 2020 11:46:42,661 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.config.AbstractConfig.logAll:279) - ConsumerConfig values:
|
||||||
|
auto.commit.interval.ms = 5000
|
||||||
|
auto.offset.reset = latest
|
||||||
|
bootstrap.servers = [192.168.40.152:9092]
|
||||||
|
check.crcs = true
|
||||||
|
client.id =
|
||||||
|
connections.max.idle.ms = 540000
|
||||||
|
default.api.timeout.ms = 60000
|
||||||
|
enable.auto.commit = false
|
||||||
|
exclude.internal.topics = true
|
||||||
|
fetch.max.bytes = 52428800
|
||||||
|
fetch.max.wait.ms = 1000
|
||||||
|
fetch.min.bytes = 1048576
|
||||||
|
group.id = lxk_0514
|
||||||
|
heartbeat.interval.ms = 3000
|
||||||
|
interceptor.classes = []
|
||||||
|
internal.leave.group.on.close = true
|
||||||
|
isolation.level = read_uncommitted
|
||||||
|
key.deserializer = class org.apache.kafka.common.serialization.StringDeserializer
|
||||||
|
max.partition.fetch.bytes = 1048576
|
||||||
|
max.poll.interval.ms = 300000
|
||||||
|
max.poll.records = 500
|
||||||
|
metadata.max.age.ms = 300000
|
||||||
|
metric.reporters = []
|
||||||
|
metrics.num.samples = 2
|
||||||
|
metrics.recording.level = INFO
|
||||||
|
metrics.sample.window.ms = 30000
|
||||||
|
partition.assignment.strategy = [class org.apache.kafka.clients.consumer.RangeAssignor]
|
||||||
|
receive.buffer.bytes = 65536
|
||||||
|
reconnect.backoff.max.ms = 1000
|
||||||
|
reconnect.backoff.ms = 50
|
||||||
|
request.timeout.ms = 30000
|
||||||
|
retry.backoff.ms = 100
|
||||||
|
sasl.client.callback.handler.class = null
|
||||||
|
sasl.jaas.config = null
|
||||||
|
sasl.kerberos.kinit.cmd = /usr/bin/kinit
|
||||||
|
sasl.kerberos.min.time.before.relogin = 60000
|
||||||
|
sasl.kerberos.service.name = null
|
||||||
|
sasl.kerberos.ticket.renew.jitter = 0.05
|
||||||
|
sasl.kerberos.ticket.renew.window.factor = 0.8
|
||||||
|
sasl.login.callback.handler.class = null
|
||||||
|
sasl.login.class = null
|
||||||
|
sasl.login.refresh.buffer.seconds = 300
|
||||||
|
sasl.login.refresh.min.period.seconds = 60
|
||||||
|
sasl.login.refresh.window.factor = 0.8
|
||||||
|
sasl.login.refresh.window.jitter = 0.05
|
||||||
|
sasl.mechanism = GSSAPI
|
||||||
|
security.protocol = PLAINTEXT
|
||||||
|
send.buffer.bytes = 131072
|
||||||
|
session.timeout.ms = 10000
|
||||||
|
ssl.cipher.suites = null
|
||||||
|
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
|
||||||
|
ssl.endpoint.identification.algorithm = https
|
||||||
|
ssl.key.password = null
|
||||||
|
ssl.keymanager.algorithm = SunX509
|
||||||
|
ssl.keystore.location = null
|
||||||
|
ssl.keystore.password = null
|
||||||
|
ssl.keystore.type = JKS
|
||||||
|
ssl.protocol = TLS
|
||||||
|
ssl.provider = null
|
||||||
|
ssl.secure.random.implementation = null
|
||||||
|
ssl.trustmanager.algorithm = PKIX
|
||||||
|
ssl.truststore.location = null
|
||||||
|
ssl.truststore.password = null
|
||||||
|
ssl.truststore.type = JKS
|
||||||
|
value.deserializer = class org.apache.kafka.common.serialization.ByteArrayDeserializer
|
||||||
|
|
||||||
|
14 May 2020 11:46:42,696 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:109) - Kafka version : 2.0.1
|
||||||
|
14 May 2020 11:46:42,697 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.common.utils.AppInfoParser$AppInfo.<init>:110) - Kafka commitId : fa14705e51bd2ce5
|
||||||
|
14 May 2020 11:46:42,700 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.KafkaChannel.createConsumerAndRecords:304) - Created new consumer to connect to Kafka
|
||||||
|
14 May 2020 11:46:42,870 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 11:46:42,871 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$FindCoordinatorResponseHandler.onSuccess:677) - [Consumer clientId=consumer-1, groupId=lxk_0514] Discovered group coordinator 192.168.40.152:9092 (id: 2147483646 rack: null)
|
||||||
|
14 May 2020 11:46:42,873 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions []
|
||||||
|
14 May 2020 11:46:42,873 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:42,887 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 18
|
||||||
|
14 May 2020 11:46:42,888 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:42,889 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:46:45,901 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 11:46:45,910 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:45,910 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 11:46:45,911 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 11:46:45,921 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 19
|
||||||
|
14 May 2020 11:46:45,922 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 11:46:45,923 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 11:51:42,641 INFO [kafka-producer-network-thread | producer-1] (org.apache.kafka.clients.Metadata.update:285) - Cluster ID: gsZT8xNUSViLGeSpG1neHg
|
||||||
|
14 May 2020 12:02:48,119 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:48,715 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:48,716 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 12:02:48,717 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:54,224 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 20
|
||||||
|
14 May 2020 12:02:54,225 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:54,225 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
|
14 May 2020 12:02:57,326 INFO [kafka-coordinator-heartbeat-thread | lxk_0514] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatResponseHandler.handle:855) - [Consumer clientId=consumer-1, groupId=lxk_0514] Attempt to heartbeat failed since group is rebalancing
|
||||||
|
14 May 2020 12:02:59,720 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinPrepare:472) - [Consumer clientId=consumer-1, groupId=lxk_0514] Revoking previously assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:59,721 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsRevoked:785) - topic CONNECTION-RECORD-LOG - partition 0 revoked.
|
||||||
|
14 May 2020 12:02:59,722 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator.sendJoinGroupRequest:509) - [Consumer clientId=consumer-1, groupId=lxk_0514] (Re-)joining group
|
||||||
|
14 May 2020 12:02:59,729 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.AbstractCoordinator$1.onSuccess:473) - [Consumer clientId=consumer-1, groupId=lxk_0514] Successfully joined group with generation 21
|
||||||
|
14 May 2020 12:02:59,730 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.onJoinComplete:280) - [Consumer clientId=consumer-1, groupId=lxk_0514] Setting newly assigned partitions [CONNECTION-RECORD-LOG-0]
|
||||||
|
14 May 2020 12:02:59,730 INFO [SinkRunner-PollingRunner-DefaultSinkProcessor] (org.apache.flume.channel.kafka.ChannelRebalanceListener.onPartitionsAssigned:792) - topic CONNECTION-RECORD-LOG - partition 0 assigned.
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user