21.11 update

2021-11-29 16:18:07 +08:00
parent 67e8a462d4
commit 669bd19fb4
3 changed files with 391 additions and 1 deletions
--- a/TSG发布版本更新记录/TSG-21.11/flink/cluster-flink-conf.yml
+++ b/TSG发布版本更新记录/TSG-21.11/flink/cluster-flink-conf.yml
@@ -1,4 +1,4 @@
-# flink集群版 配置文件增加checkpoint history
+# flink集群版 flink-conf.yml配置文件增加checkpoint history

 #==============================================================================
 #checkpoint
--- a/TSG发布版本更新记录/TSG-21.11/flink/docker-compose.yml
+++ b/TSG发布版本更新记录/TSG-21.11/flink/docker-compose.yml
@@ -0,0 +1,40 @@
+
+version: '2'
+
+services:
+  jobmanager:
+    image: flink:1.13.1
+    restart: always
+    container_name: jobmanager
+    environment:
+      START_TYPE: "jobmanager"
+      JOBMANAGER_IP: "192.168.40.203"
+    volumes:
+      - "/home/galaxy/volumes/flink/jobmanager/conf/flink-conf.yaml:/opt/flink-1.13.1/conf/flink-conf.yaml"
+      - "/home/galaxy/volumes/flink/jobmanager/log:/opt/flink-1.13.1/log"
+      - "/home/galaxy/volumes/flink/completed-jobs:/opt/flink-1.13.1/completed-jobs"
+      - "/home/galaxy/volumes/flink/checkpoint:/opt/flink-1.13.1/checkpoint"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "/etc/timezone:/etc/timezone:ro"
+    network_mode: "host"
+
+
+
+version: '2'
+services:
+  taskmanager:
+    image: flink:1.13.1
+    restart: always
+    container_name: taskmanager
+    environment:
+      START_TYPE: "taskmanager"
+      JOBMANAGER_IP: "192.168.40.203"
+    volumes:
+      - "/home/galaxy/volumes/flink/taskmanager/conf/flink-conf.yaml:/opt/flink-1.13.1/conf/flink-conf.yaml"
+      - "/home/galaxy/volumes/flink/taskmanager/log:/opt/flink-1.13.1/log"
+      - "/home/galaxy/workspace/topology/dat/:/opt/dat/"
+      - "/home/galaxy/volumes/flink/completed-jobs:/opt/flink-1.13.1/completed-jobs"
+      - "/home/galaxy/volumes/flink/checkpoint:/opt/flink-1.13.1/checkpoint"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "/etc/timezone:/etc/timezone:ro"
+    network_mode: "host"
--- a/TSG发布版本更新记录/TSG-21.11/flink/standalone-flink.conf.yml
+++ b/TSG发布版本更新记录/TSG-21.11/flink/standalone-flink.conf.yml
@@ -0,0 +1,350 @@
+#  censed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+#==============================================================================
+# Common
+#==============================================================================
+
+# The external address of the host on which the JobManager runs and can be
+# reached by the TaskManagers and any clients which want to connect. This setting
+# is only used in Standalone mode and may be overwritten on the JobManager side
+# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
+# In high availability mode, if you use the bin/start-cluster.sh script and setup
+# the conf/masters file, this will be taken care of automatically. Yarn/Mesos
+# automatically configure the host name based on the hostname of the node where the
+# JobManager runs.
+
+jobmanager.rpc.address: {{ machine_host }}
+
+# The RPC port where the JobManager is reachable.
+
+env.java.opts: "-XX:+UseG1GC -XX:NewRatio=2 -XX:MaxGCPauseMillis=300 -XX:InitiatingHeapOccupancyPercent=35 -Xloggc:/opt/flink-1.13.1/log/gc.log -XX:+PrintGCDetails -XX:-OmitStackTraceInFastThrow -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=20 -XX:GCLogFileSize=20M"
+
+jobmanager.rpc.port: 6123
+
+#开启负载均衡
+cluster.evenly-spread-out-slots: true
+# The total process memory size for the JobManager.
+#
+# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
+
+#jobmanager.memory.process.size: 1600m
+#taskmanager.heap.mb: 2g
+#jobmanager.heap.mb: 1g
+
+#托管内存大小-内存管理器管理的堆外内存，为排序，哈希，中间结果缓存预留
+taskmanager.memory.managed.size: 512M
+
+#网络缓冲区大小，可指定max、min的具体大小；也可使用fraction，默认占flink内存的0.1,两者使用一个即可
+taskmanager.memory.network.fraction: 0.1
+
+#taskexcutor的堆外内存大小
+taskmanager.memory.framework.off-heap.size: 512M
+taskmanager.memory.framework.heap.size: 512M
+taskmanager.memory.framework.off-heap.size: 512M
+
+#jvm元空间大小
+taskmanager.memory.jvm-metaspace.size: 740M
+taskmanager.memory.framework.off-heap.batch-shuffle.size: 128M
+
+jobmanager.memory.jvm-metaspace.size: 512M
+taskmanager.network.sort-shuffle.min-buffers: 256
+#输入/输出额外网络缓冲区的个数，默认8，缓解由于子分区之间的数据分布不平衡造成的背压。
+#如果节点之间的往返时间和/或集群中的机器数量更多，则应该增加该值。
+taskmanager.network.memory.floating-buffers-per-gate: 8
+#网络缓冲区通道数量，默认为2，最少为2
+taskmanager.network.memory.buffers-per-channel: 2
+taskmanager.network.memory.max-buffers-per-channel: 16
+
+#flink进程总内存，process.size与flink.size选择其中一个。docker使用process.size
+taskmanager.memory.process.size: 7168M
+jobmanager.memory.flink.size: 1024M
+# The total process memory size for the TaskManager.
+#
+# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
+
+#taskmanager.memory.process.size: 1728m
+
+# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
+# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
+#
+# taskmanager.memory.flink.size: 1280m
+
+# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
+
+taskmanager.numberOfTaskSlots: 120
+
+
+# The parallelism used for programs that did not specify and other parallelism.
+
+parallelism.default: 60
+
+# The default file system scheme and authority.
+#
+# By default file paths without scheme are interpreted relative to the local
+# root file system 'file:///'. Use this to override the default and interpret
+# relative paths relative to a different file system,
+# for example 'hdfs://mynamenode:12345'
+#
+# fs.default-scheme
+
+#==============================================================================
+# High Availability
+#==============================================================================
+
+# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
+#
+# high-availability: zookeeper
+
+# The path where metadata for master recovery is persisted. While ZooKeeper stores
+# the small ground truth for checkpoint and leader election, this location stores
+# the larger objects, like persisted dataflow graphs.
+#
+# Must be a durable file system that is accessible from all nodes
+# (like HDFS, S3, Ceph, nfs, ...)
+#
+# high-availability.storageDir: hdfs:///flink/ha/
+
+# The list of ZooKeeper quorum peers that coordinate the high-availability
+# setup. This must be a list of the form:
+# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
+#
+
+#向akka发送心跳超时时间设置及间隔
+heartbeat.timeout: 180000
+heartbeat.interval: 20000
+akka.ask.timeout: 300 s
+
+# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
+# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
+# The default value is "open" and it can be changed to "creator" if ZK security is enabled
+#
+# high-availability.zookeeper.client.acl: open
+
+
+#Savepoint:
+#state.savepoints.dir: hdfs://192.168.44.11:9000/flink/savepoints
+
+#==============================================================================
+# Fault tolerance and checkpointing
+#==============================================================================
+
+# The backend that will be used to store operator state checkpoints if
+# checkpointing is enabled.
+#
+# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the
+# <class-name-of-factory>.
+#
+# state.backend: filesystem
+
+# Directory for checkpoints filesystem, when using any of the default bundled
+# state backends.
+#
+# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints
+
+# Default target directory for savepoints, optional.
+#
+# state.savepoints.dir: hdfs://namenode-host:port/flink-savepoints
+
+# Flag to enable/disable incremental checkpoints for backends that
+# support incremental checkpoints (like the RocksDB state backend).
+#
+# state.backend.incremental: false
+
+# The failover strategy, i.e., how the job computation recovers from task failures.
+# Only restart tasks that may have been affected by the task failure, which typically includes
+# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
+#==============================================================================
+# checkpoint
+#==============================================================================
+state.checkpoints.dir: file:///opt/flink-1.13.1/checkpoint
+##检查点间隔
+execution.checkpointing.interval: 300000
+#
+##检查点的超时设置
+execution.checkpointing.timeout: 10min
+#
+###检查点的个数
+state.checkpoints.num-retained: 10
+
+state.backend: filesystem
+
+#==============================================================================
+# historyserver
+#==============================================================================
+
+# 上传已完成作业信息的目录
+jobmanager.archive.fs.dir: file:///opt/flink-1.13.1/completed-jobs
+
+# 监视以下目录中已完成的作业
+historyserver.archive.fs.dir: file:///opt/flink-1.13.1/completed-jobs
+
+# 每 10 秒刷新一次
+historyserver.archive.fs.refresh-interval: 10000
+
+#完成的作业到期并从作业存储区中清除的秒数
+jobstore.expiration-time: 7200
+
+#避免由于task不能正常取消而使taskmanager服务挂掉
+task.cancellation.timeout: 0
+
+#任务重启策略
+jobmanager.execution.failover-strategy: region
+rest.port: 8080
+restart-strategy: fixed-delay
+
+#任务重启次数及间隔时间
+restart-strategy.fixed-delay.attempts: 100
+restart-strategy.fixed-delay.delay: 180 s
+
+jobmanager.web.upload.dir: /opt/flink-1.13.1/flink-web
+#==============================================================================
+# Flink metrics
+#==============================================================================
+
+#监控class
+metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
+
+#监控暴露的端口，默认9249，更改为9909
+metrics.reporter.prom.port: 9909-9910
+
+  #==============================================================================
+  # Rest & web frontend
+  #==============================================================================
+
+  # The port to which the REST client connects to. If rest.bind-port has
+  # not been specified, then the server will bind to this port as well.
+  #
+  #rest.port: 8092
+
+  # The address to which the REST client will connect to
+  #
+  #rest.address: 0.0.0.0
+
+  # Port range for the REST and web server to bind to.
+  #
+  #rest.bind-port: 8080-8090
+
+  # The address that the REST & web server binds to
+  #
+  #rest.bind-address: 0.0.0.0
+
+  # Flag to specify whether job submission is enabled from the web-based
+  # runtime monitor. Uncomment to disable.
+
+  #web.submit.enable: false
+
+  #==============================================================================
+  # Advanced
+  #==============================================================================
+
+  # Override the directories for temporary files. If not specified, the
+  # system-specific Java temporary directory (java.io.tmpdir property) is taken.
+  #
+  # For framework setups on Yarn or Mesos, Flink will automatically pick up the
+  # containers' temp directories without any need for configuration.
+  #
+  # Add a delimited list for multiple directories, using the system directory
+  # delimiter (colon ':' on unix) or a comma, e.g.:
+  #     /data1/tmp:/data2/tmp:/data3/tmp
+  #
+  # Note: Each directory entry is read from and written to by a different I/O
+  # thread. You can include the same directory multiple times in order to create
+  # multiple I/O threads against that directory. This is for example relevant for
+  # high-throughput RAIDs.
+  #
+  # io.tmp.dirs: /tmp
+
+  # The classloading resolve order. Possible values are 'child-first' (Flink's default)
+  # and 'parent-first' (Java's default).
+  #
+  # Child first classloading allows users to use different dependency/library
+  # versions in their application than those in the classpath. Switching back
+  # to 'parent-first' may help with debugging dependency issues.
+  #
+
+  #类加载方式
+  # classloader.resolve-order: child-first
+  classloader.resolve-order: parent-first
+
+# The amount of memory going to the network stack. These numbers usually need
+# no tuning. Adjusting them may be necessary in case of an "Insufficient number
+# of network buffers" error. The default min is 64MB, the default max is 1GB.
+#
+# taskmanager.memory.network.fraction: 0.1
+# taskmanager.memory.network.min: 64mb
+# taskmanager.memory.network.max: 1gb
+
+#==============================================================================
+# Flink Cluster Security Configuration
+#==============================================================================
+
+# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
+# may be enabled in four steps:
+# 1. configure the local krb5.conf file
+# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
+# 3. make the credentials available to various JAAS login contexts
+# 4. configure the connector to use JAAS/SASL
+
+# The below configure how Kerberos credentials are provided. A keytab will be used instead of
+# a ticket cache if the keytab path and principal are set.
+
+# security.kerberos.login.use-ticket-cache: true
+# security.kerberos.login.keytab: /path/to/kerberos/keytab
+# security.kerberos.login.principal: flink-user
+
+# The configuration below defines which JAAS login contexts
+
+# security.kerberos.login.contexts: Client,KafkaClient
+
+#==============================================================================
+# ZK Security Configuration
+#==============================================================================
+
+# Below configurations are applicable if ZK ensemble is configured for security
+
+# Override below configuration to provide custom ZK service name if configured
+# zookeeper.sasl.service-name: zookeeper
+
+# The configuration below must match one of the values set in "security.kerberos.login.contexts"
+# zookeeper.sasl.login-context-name: Client
+
+#==============================================================================
+# HistoryServer
+#==============================================================================
+
+# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
+
+# Directory to upload completed jobs to. Add this directory to the list of
+# monitored directories of the HistoryServer as well (see below).
+#jobmanager.archive.fs.dir: hdfs:///completed-jobs/
+
+# The address under which the web-based HistoryServer listens.
+#historyserver.web.address: 0.0.0.0
+
+# The port under which the web-based HistoryServer listens.
+#historyserver.web.port: 8082
+
+# Comma separated list of directories to monitor for completed jobs.
+#historyserver.archive.fs.dir: hdfs:///completed-jobs/
+#historyserver.archive.fs.dir: /opt/flink-1.13.1/completed-jobs/
+
+# Interval in milliseconds for refreshing the monitored directories.
+#historyserver.archive.fs.refresh-interval: 10000
+
+