diff --git a/TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml b/TSG发布版本更新记录/TSG-21.11/flink/cluster-flink-conf.yml similarity index 94% rename from TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml rename to TSG发布版本更新记录/TSG-21.11/flink/cluster-flink-conf.yml index af3f6ca..5c08727 100644 --- a/TSG发布版本更新记录/TSG-21.11/flink/flink-conf.yml +++ b/TSG发布版本更新记录/TSG-21.11/flink/cluster-flink-conf.yml @@ -1,4 +1,4 @@ -# flink集群版 配置文件增加checkpoint history +# flink集群版 flink-conf.yml配置文件增加checkpoint history #============================================================================== #checkpoint diff --git a/TSG发布版本更新记录/TSG-21.11/flink/docker-compose.yml b/TSG发布版本更新记录/TSG-21.11/flink/docker-compose.yml new file mode 100644 index 0000000..edc7d9f --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/flink/docker-compose.yml @@ -0,0 +1,40 @@ + +version: '2' + +services: + jobmanager: + image: flink:1.13.1 + restart: always + container_name: jobmanager + environment: + START_TYPE: "jobmanager" + JOBMANAGER_IP: "192.168.40.203" + volumes: + - "/home/galaxy/volumes/flink/jobmanager/conf/flink-conf.yaml:/opt/flink-1.13.1/conf/flink-conf.yaml" + - "/home/galaxy/volumes/flink/jobmanager/log:/opt/flink-1.13.1/log" + - "/home/galaxy/volumes/flink/completed-jobs:/opt/flink-1.13.1/completed-jobs" + - "/home/galaxy/volumes/flink/checkpoint:/opt/flink-1.13.1/checkpoint" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + network_mode: "host" + + + +version: '2' +services: + taskmanager: + image: flink:1.13.1 + restart: always + container_name: taskmanager + environment: + START_TYPE: "taskmanager" + JOBMANAGER_IP: "192.168.40.203" + volumes: + - "/home/galaxy/volumes/flink/taskmanager/conf/flink-conf.yaml:/opt/flink-1.13.1/conf/flink-conf.yaml" + - "/home/galaxy/volumes/flink/taskmanager/log:/opt/flink-1.13.1/log" + - "/home/galaxy/workspace/topology/dat/:/opt/dat/" + - "/home/galaxy/volumes/flink/completed-jobs:/opt/flink-1.13.1/completed-jobs" + - "/home/galaxy/volumes/flink/checkpoint:/opt/flink-1.13.1/checkpoint" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + network_mode: "host" \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.11/flink/standalone-flink.conf.yml b/TSG发布版本更新记录/TSG-21.11/flink/standalone-flink.conf.yml new file mode 100644 index 0000000..3325995 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.11/flink/standalone-flink.conf.yml @@ -0,0 +1,350 @@ +# censed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +#============================================================================== +# Common +#============================================================================== + +# The external address of the host on which the JobManager runs and can be +# reached by the TaskManagers and any clients which want to connect. This setting +# is only used in Standalone mode and may be overwritten on the JobManager side +# by specifying the --host parameter of the bin/jobmanager.sh executable. +# In high availability mode, if you use the bin/start-cluster.sh script and setup +# the conf/masters file, this will be taken care of automatically. Yarn/Mesos +# automatically configure the host name based on the hostname of the node where the +# JobManager runs. + +jobmanager.rpc.address: {{ machine_host }} + +# The RPC port where the JobManager is reachable. + +env.java.opts: "-XX:+UseG1GC -XX:NewRatio=2 -XX:MaxGCPauseMillis=300 -XX:InitiatingHeapOccupancyPercent=35 -Xloggc:/opt/flink-1.13.1/log/gc.log -XX:+PrintGCDetails -XX:-OmitStackTraceInFastThrow -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=20 -XX:GCLogFileSize=20M" + +jobmanager.rpc.port: 6123 + +#开启负载均衡 +cluster.evenly-spread-out-slots: true +# The total process memory size for the JobManager. +# +# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead. + +#jobmanager.memory.process.size: 1600m +#taskmanager.heap.mb: 2g +#jobmanager.heap.mb: 1g + +#托管内存大小-内存管理器管理的堆外内存,为排序,哈希,中间结果缓存预留 +taskmanager.memory.managed.size: 512M + +#网络缓冲区大小,可指定max、min的具体大小;也可使用fraction,默认占flink内存的0.1,两者使用一个即可 +taskmanager.memory.network.fraction: 0.1 + +#taskexcutor的堆外内存大小 +taskmanager.memory.framework.off-heap.size: 512M +taskmanager.memory.framework.heap.size: 512M +taskmanager.memory.framework.off-heap.size: 512M + +#jvm元空间大小 +taskmanager.memory.jvm-metaspace.size: 740M +taskmanager.memory.framework.off-heap.batch-shuffle.size: 128M + +jobmanager.memory.jvm-metaspace.size: 512M +taskmanager.network.sort-shuffle.min-buffers: 256 +#输入/输出额外网络缓冲区的个数,默认8,缓解由于子分区之间的数据分布不平衡造成的背压。 +#如果节点之间的往返时间和/或集群中的机器数量更多,则应该增加该值。 +taskmanager.network.memory.floating-buffers-per-gate: 8 +#网络缓冲区通道数量,默认为2,最少为2 +taskmanager.network.memory.buffers-per-channel: 2 +taskmanager.network.memory.max-buffers-per-channel: 16 + +#flink进程总内存,process.size与flink.size选择其中一个。docker使用process.size +taskmanager.memory.process.size: 7168M +jobmanager.memory.flink.size: 1024M +# The total process memory size for the TaskManager. +# +# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead. + +#taskmanager.memory.process.size: 1728m + +# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'. +# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory. +# +# taskmanager.memory.flink.size: 1280m + +# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline. + +taskmanager.numberOfTaskSlots: 120 + + +# The parallelism used for programs that did not specify and other parallelism. + +parallelism.default: 60 + +# The default file system scheme and authority. +# +# By default file paths without scheme are interpreted relative to the local +# root file system 'file:///'. Use this to override the default and interpret +# relative paths relative to a different file system, +# for example 'hdfs://mynamenode:12345' +# +# fs.default-scheme + +#============================================================================== +# High Availability +#============================================================================== + +# The high-availability mode. Possible options are 'NONE' or 'zookeeper'. +# +# high-availability: zookeeper + +# The path where metadata for master recovery is persisted. While ZooKeeper stores +# the small ground truth for checkpoint and leader election, this location stores +# the larger objects, like persisted dataflow graphs. +# +# Must be a durable file system that is accessible from all nodes +# (like HDFS, S3, Ceph, nfs, ...) +# +# high-availability.storageDir: hdfs:///flink/ha/ + +# The list of ZooKeeper quorum peers that coordinate the high-availability +# setup. This must be a list of the form: +# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181) +# + +#向akka发送心跳超时时间设置及间隔 +heartbeat.timeout: 180000 +heartbeat.interval: 20000 +akka.ask.timeout: 300 s + +# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes +# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE) +# The default value is "open" and it can be changed to "creator" if ZK security is enabled +# +# high-availability.zookeeper.client.acl: open + + +#Savepoint: +#state.savepoints.dir: hdfs://192.168.44.11:9000/flink/savepoints + +#============================================================================== +# Fault tolerance and checkpointing +#============================================================================== + +# The backend that will be used to store operator state checkpoints if +# checkpointing is enabled. +# +# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the +# . +# +# state.backend: filesystem + +# Directory for checkpoints filesystem, when using any of the default bundled +# state backends. +# +# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints + +# Default target directory for savepoints, optional. +# +# state.savepoints.dir: hdfs://namenode-host:port/flink-savepoints + +# Flag to enable/disable incremental checkpoints for backends that +# support incremental checkpoints (like the RocksDB state backend). +# +# state.backend.incremental: false + +# The failover strategy, i.e., how the job computation recovers from task failures. +# Only restart tasks that may have been affected by the task failure, which typically includes +# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption. +#============================================================================== +# checkpoint +#============================================================================== +state.checkpoints.dir: file:///opt/flink-1.13.1/checkpoint +##检查点间隔 +execution.checkpointing.interval: 300000 +# +##检查点的超时设置 +execution.checkpointing.timeout: 10min +# +###检查点的个数 +state.checkpoints.num-retained: 10 + +state.backend: filesystem + +#============================================================================== +# historyserver +#============================================================================== + +# 上传已完成作业信息的目录 +jobmanager.archive.fs.dir: file:///opt/flink-1.13.1/completed-jobs + +# 监视以下目录中已完成的作业 +historyserver.archive.fs.dir: file:///opt/flink-1.13.1/completed-jobs + +# 每 10 秒刷新一次 +historyserver.archive.fs.refresh-interval: 10000 + +#完成的作业到期并从作业存储区中清除的秒数 +jobstore.expiration-time: 7200 + +#避免由于task不能正常取消而使taskmanager服务挂掉 +task.cancellation.timeout: 0 + +#任务重启策略 +jobmanager.execution.failover-strategy: region +rest.port: 8080 +restart-strategy: fixed-delay + +#任务重启次数及间隔时间 +restart-strategy.fixed-delay.attempts: 100 +restart-strategy.fixed-delay.delay: 180 s + +jobmanager.web.upload.dir: /opt/flink-1.13.1/flink-web +#============================================================================== +# Flink metrics +#============================================================================== + +#监控class +metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + +#监控暴露的端口,默认9249,更改为9909 +metrics.reporter.prom.port: 9909-9910 + + #============================================================================== + # Rest & web frontend + #============================================================================== + + # The port to which the REST client connects to. If rest.bind-port has + # not been specified, then the server will bind to this port as well. + # + #rest.port: 8092 + + # The address to which the REST client will connect to + # + #rest.address: 0.0.0.0 + + # Port range for the REST and web server to bind to. + # + #rest.bind-port: 8080-8090 + + # The address that the REST & web server binds to + # + #rest.bind-address: 0.0.0.0 + + # Flag to specify whether job submission is enabled from the web-based + # runtime monitor. Uncomment to disable. + + #web.submit.enable: false + + #============================================================================== + # Advanced + #============================================================================== + + # Override the directories for temporary files. If not specified, the + # system-specific Java temporary directory (java.io.tmpdir property) is taken. + # + # For framework setups on Yarn or Mesos, Flink will automatically pick up the + # containers' temp directories without any need for configuration. + # + # Add a delimited list for multiple directories, using the system directory + # delimiter (colon ':' on unix) or a comma, e.g.: + # /data1/tmp:/data2/tmp:/data3/tmp + # + # Note: Each directory entry is read from and written to by a different I/O + # thread. You can include the same directory multiple times in order to create + # multiple I/O threads against that directory. This is for example relevant for + # high-throughput RAIDs. + # + # io.tmp.dirs: /tmp + + # The classloading resolve order. Possible values are 'child-first' (Flink's default) + # and 'parent-first' (Java's default). + # + # Child first classloading allows users to use different dependency/library + # versions in their application than those in the classpath. Switching back + # to 'parent-first' may help with debugging dependency issues. + # + + #类加载方式 + # classloader.resolve-order: child-first + classloader.resolve-order: parent-first + +# The amount of memory going to the network stack. These numbers usually need +# no tuning. Adjusting them may be necessary in case of an "Insufficient number +# of network buffers" error. The default min is 64MB, the default max is 1GB. +# +# taskmanager.memory.network.fraction: 0.1 +# taskmanager.memory.network.min: 64mb +# taskmanager.memory.network.max: 1gb + +#============================================================================== +# Flink Cluster Security Configuration +#============================================================================== + +# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors - +# may be enabled in four steps: +# 1. configure the local krb5.conf file +# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit) +# 3. make the credentials available to various JAAS login contexts +# 4. configure the connector to use JAAS/SASL + +# The below configure how Kerberos credentials are provided. A keytab will be used instead of +# a ticket cache if the keytab path and principal are set. + +# security.kerberos.login.use-ticket-cache: true +# security.kerberos.login.keytab: /path/to/kerberos/keytab +# security.kerberos.login.principal: flink-user + +# The configuration below defines which JAAS login contexts + +# security.kerberos.login.contexts: Client,KafkaClient + +#============================================================================== +# ZK Security Configuration +#============================================================================== + +# Below configurations are applicable if ZK ensemble is configured for security + +# Override below configuration to provide custom ZK service name if configured +# zookeeper.sasl.service-name: zookeeper + +# The configuration below must match one of the values set in "security.kerberos.login.contexts" +# zookeeper.sasl.login-context-name: Client + +#============================================================================== +# HistoryServer +#============================================================================== + +# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop) + +# Directory to upload completed jobs to. Add this directory to the list of +# monitored directories of the HistoryServer as well (see below). +#jobmanager.archive.fs.dir: hdfs:///completed-jobs/ + +# The address under which the web-based HistoryServer listens. +#historyserver.web.address: 0.0.0.0 + +# The port under which the web-based HistoryServer listens. +#historyserver.web.port: 8082 + +# Comma separated list of directories to monitor for completed jobs. +#historyserver.archive.fs.dir: hdfs:///completed-jobs/ +#historyserver.archive.fs.dir: /opt/flink-1.13.1/completed-jobs/ + +# Interval in milliseconds for refreshing the monitored directories. +#historyserver.archive.fs.refresh-interval: 10000 + +