199 lines
8.7 KiB
Plaintext
199 lines
8.7 KiB
Plaintext
|
################################################################################
|
|||
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|||
|
|
# or more contributor license agreements. See the NOTICE file
|
|||
|
|
# distributed with this work for additional information
|
|||
|
|
# regarding copyright ownership. The ASF licenses this file
|
|||
|
|
# to you under the Apache License, Version 2.0 (the
|
|||
|
|
# "License"); you may not use this file except in compliance
|
|||
|
|
# with the License. You may obtain a copy of the License at
|
|||
|
|
#
|
|||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
|
#
|
|||
|
|
# Unless required by applicable law or agreed to in writing, software
|
|||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
|
# See the License for the specific language governing permissions and
|
|||
|
|
# limitations under the License.
|
|||
|
|
################################################################################
|
|||
|
|
|
|||
|
|
|
|||
|
|
#==============================================================================
|
|||
|
|
# Common
|
|||
|
|
#==============================================================================
|
|||
|
|
|
|||
|
|
# The external address of the host on which the JobManager runs and can be
|
|||
|
|
# reached by the TaskManagers and any clients which want to connect. This setting
|
|||
|
|
# is only used in Standalone mode and may be overwritten on the JobManager side
|
|||
|
|
# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
|
|||
|
|
# In high availability mode, if you use the bin/start-cluster.sh script and setup
|
|||
|
|
# the conf/masters file, this will be taken care of automatically. Yarn/Mesos
|
|||
|
|
# automatically configure the host name based on the hostname of the node where the
|
|||
|
|
# JobManager runs.
|
|||
|
|
|
|||
|
|
jobmanager.rpc.address: {{ groups.yarn[0] }}
|
|||
|
|
|
|||
|
|
#jobmanager rpc 端口
|
|||
|
|
jobmanager.rpc.port: 6123
|
|||
|
|
|
|||
|
|
#允许任务在所有taskmanager上均匀分布
|
|||
|
|
cluster.evenly-spread-out-slots: true
|
|||
|
|
|
|||
|
|
#避免报出metaspace oom而是flink jvm进程挂掉
|
|||
|
|
classloader.fail-on-metaspace-oom-error: false
|
|||
|
|
|
|||
|
|
#规避第三方库堆栈泄漏问题
|
|||
|
|
classloader.check-leaked-classloader: false
|
|||
|
|
|
|||
|
|
#避免由于task不能正常取消而使taskmanager服务挂掉
|
|||
|
|
task.cancellation.timeout: 0
|
|||
|
|
|
|||
|
|
#JobManager进程占用的所有与Flink相关的内存
|
|||
|
|
jobmanager.memory.process.size: {{ flink['jobmanager.memory.process.size'] }}
|
|||
|
|
|
|||
|
|
#TaskManager进程占用的所有与Flink相关的内存
|
|||
|
|
taskmanager.memory.process.size: {{ flink['taskmanager.memory.process.size'] }}
|
|||
|
|
|
|||
|
|
#taskmanager使用的堆外内存的大小
|
|||
|
|
taskmanager.memory.managed.size: 128M
|
|||
|
|
|
|||
|
|
#taskmanager.memory.off-heap默认为false，主要指的是Flink Managed Memory使用Heap还是Non-heap，
|
|||
|
|
#默认使用Heap，如果开启使用Non-heap将再减少一部分资源
|
|||
|
|
taskmanager.memory.off-heap: false
|
|||
|
|
|
|||
|
|
#堆外部分（Framework Off-Heap），以直接内存形式分配
|
|||
|
|
taskmanager.memory.framework.off-heap.size: {{ flink['taskmanager.memory.framework.off-heap.size'] }}
|
|||
|
|
|
|||
|
|
#taskmanager元数据大小 默认256M
|
|||
|
|
taskmanager.memory.jvm-metaspace.size: {{ flink['taskmanager.memory.jvm-metaspace.size'] }}
|
|||
|
|
|
|||
|
|
#每个排序合并阻塞结果分区所需的最小网络缓冲区数，默认64。对于生产使用，建议将该配置值增加到2048，以提高数据压缩比并减少较小的网络数据包。增加该参数值，需要增加总网络内存大小。
|
|||
|
|
taskmanager.network.sort-shuffle.min-buffers: 64
|
|||
|
|
|
|||
|
|
#用于读取shuffle数据的内存大小（目前只用于排序合并shuffle）。该内存参数占用framework.off-heap.size内存，默认32M，当更改该参数时，需要增加framework.off-heap.size内存大小。
|
|||
|
|
taskmanager.memory.framework.off-heap.batch-shuffle.size: 8M
|
|||
|
|
|
|||
|
|
#每个通道可以使用的最大缓冲区数，默认为10。该参数可以通过防止在数据倾斜和配置的浮动缓冲区数量高的情况下缓冲的动态数据的过度增长来加速检查点对齐。
|
|||
|
|
taskmanager.network.memory.max-buffers-per-channel: 10
|
|||
|
|
|
|||
|
|
# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
|
|||
|
|
taskmanager.numberOfTaskSlots: {{ flink['taskmanager.numberOfTaskSlots'] }}
|
|||
|
|
|
|||
|
|
# The parallelism used for programs that did not specify and other parallelism.
|
|||
|
|
parallelism.default: 1
|
|||
|
|
|
|||
|
|
# The default file system scheme and authority.
|
|||
|
|
#
|
|||
|
|
# By default file paths without scheme are interpreted relative to the local
|
|||
|
|
# root file system 'file:///'. Use this to override the default and interpret
|
|||
|
|
# relative paths relative to a different file system,
|
|||
|
|
# for example 'hdfs://mynamenode:12345'
|
|||
|
|
#
|
|||
|
|
# fs.default-scheme
|
|||
|
|
|
|||
|
|
#==============================================================================
|
|||
|
|
# NetWork
|
|||
|
|
#==============================================================================
|
|||
|
|
|
|||
|
|
#网络缓冲区数目，默认为8。帮助缓解由于子分区之间的数据分布不均匀造成的背压。
|
|||
|
|
taskmanager.network.memory.floating-buffers-per-gate: 8
|
|||
|
|
|
|||
|
|
#输入/输出通道使用的独占网络缓冲区的数量。至少配置2。
|
|||
|
|
taskmanager.network.memory.buffers-per-channel: 2
|
|||
|
|
|
|||
|
|
#用于TaskManager之间（shuffle、广播等）及与外部组件的数据传输
|
|||
|
|
#Min
|
|||
|
|
taskmanager.memory.network.min: 128M
|
|||
|
|
#Max
|
|||
|
|
taskmanager.memory.network.max: {{ flink['taskmanager.memory.network.max'] }}
|
|||
|
|
|
|||
|
|
#==============================================================================
|
|||
|
|
# High Availability
|
|||
|
|
#==============================================================================
|
|||
|
|
|
|||
|
|
# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
|
|||
|
|
#
|
|||
|
|
# high-availability: zookeeper
|
|||
|
|
|
|||
|
|
# The path where metadata for master recovery is persisted. While ZooKeeper stores
|
|||
|
|
# the small ground truth for checkpoint and leader election, this location stores
|
|||
|
|
# the larger objects, like persisted dataflow graphs.
|
|||
|
|
#
|
|||
|
|
# Must be a durable file system that is accessible from all nodes
|
|||
|
|
# (like HDFS, S3, Ceph, nfs, ...)
|
|||
|
|
#
|
|||
|
|
# high-availability.storageDir: hdfs:///flink/ha/
|
|||
|
|
|
|||
|
|
# The list of ZooKeeper quorum peers that coordinate the high-availability
|
|||
|
|
# setup. This must be a list of the form:
|
|||
|
|
# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
|
|||
|
|
{% if groups.yarn | length > 1 %}
|
|||
|
|
state.checkpoints.dir: hdfs:///flink/checkpoint/
|
|||
|
|
{% elif groups.yarn | length == 1 %}
|
|||
|
|
state.checkpoints.dir: file://{{ deploy_dir }}/{{ hadoop_version }}/{{ flink_version }}/checkpoint
|
|||
|
|
{% endif %}
|
|||
|
|
|
|||
|
|
heartbeat.timeout: 180000
|
|||
|
|
|
|||
|
|
heartbeat.interval: 20000
|
|||
|
|
|
|||
|
|
akka.ask.timeout: 300 s
|
|||
|
|
|
|||
|
|
# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
|
|||
|
|
# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
|
|||
|
|
# The default value is "open" and it can be changed to "creator" if ZK security is enabled
|
|||
|
|
#
|
|||
|
|
# high-availability.zookeeper.client.acl: open
|
|||
|
|
|
|||
|
|
# The failover strategy, i.e., how the job computation recovers from task failures.
|
|||
|
|
# Only restart tasks that may have been affected by the task failure, which typically includes
|
|||
|
|
# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
|
|||
|
|
jobmanager.execution.failover-strategy: region
|
|||
|
|
|
|||
|
|
restart-strategy: fixed-delay
|
|||
|
|
|
|||
|
|
restart-strategy.fixed-delay.attempts: 2147483647
|
|||
|
|
|
|||
|
|
yarn.application-attempts: 10000
|
|||
|
|
|
|||
|
|
restart-strategy.fixed-delay.delay: 5 s
|
|||
|
|
|
|||
|
|
web.submit.enable: false
|
|||
|
|
|
|||
|
|
#==============================================================================
|
|||
|
|
# Advanced
|
|||
|
|
#==============================================================================
|
|||
|
|
|
|||
|
|
# Override the directories for temporary files. If not specified, the
|
|||
|
|
# system-specific Java temporary directory (java.io.tmpdir property) is taken.
|
|||
|
|
#
|
|||
|
|
# For framework setups on Yarn or Mesos, Flink will automatically pick up the
|
|||
|
|
# containers' temp directories without any need for configuration.
|
|||
|
|
#
|
|||
|
|
# Add a delimited list for multiple directories, using the system directory
|
|||
|
|
# delimiter (colon ':' on unix) or a comma, e.g.:
|
|||
|
|
# /data1/tmp:/data2/tmp:/data3/tmp
|
|||
|
|
#
|
|||
|
|
# Note: Each directory entry is read from and written to by a different I/O
|
|||
|
|
# thread. You can include the same directory multiple times in order to create
|
|||
|
|
# multiple I/O threads against that directory. This is for example relevant for
|
|||
|
|
# high-throughput RAIDs.
|
|||
|
|
#
|
|||
|
|
# io.tmp.dirs: /tmp
|
|||
|
|
|
|||
|
|
# The classloading resolve order. Possible values are 'child-first' (Flink's default)
|
|||
|
|
# and 'parent-first' (Java's default).
|
|||
|
|
#
|
|||
|
|
# Child first classloading allows users to use different dependency/library
|
|||
|
|
# versions in their application than those in the classpath. Switching back
|
|||
|
|
# to 'parent-first' may help with debugging dependency issues.
|
|||
|
|
#
|
|||
|
|
# classloader.resolve-order: child-first
|
|||
|
|
classloader.resolve-order: parent-first
|
|||
|
|
|
|||
|
|
metrics.reporter.promgateway.class: org.apache.flink.metrics.prometheus.PrometheusPushGatewayReporter
|
|||
|
|
metrics.reporter.promgateway.randomJobNameSuffix: true
|
|||
|
|
metrics.reporter.promgateway.deleteOnShutdown: true
|
|||
|
|
metrics.reporter.promgateway.interval: 10 SECONDS
|
|||
|
|
metrics.reporter.promgateway.host: 127.0.0.1
|
|||
|
|
metrics.reporter.promgateway.port: 9091
|