1、重构代码，改用quartz做线程调度，实现HA
2、多个库的 向同一个库的表 同步数据 在同一个线程执行，保证不会造成死锁的情况，同时不会因为一个表同步事件过长而影响同一个库的其它表同步 3、部分特殊表通过 拦截器的方式 实现同步，如detection_set_info,meta_data。拦截器信息已配置到 table_sync_info 表中 4、重新整理nms 分库与主库 初始化语句及 同步配置表，初始化时 node_table.sync_status,event_record_library.db_id 的默认值需要修改为所在库的 id,id保存在sync_db_info 中，确保所有同步库 sync_db_info 表信息一致
This commit is contained in:
fangshunjian
@@ -2,13 +2,17 @@
|
||||
<classpath>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry kind="src" path="conf"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/jdk1.7.0_80"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-1.2.15.jar"/>
|
||||
<classpathentry kind="lib" path="lib/fastjson-1.2.47.jar"/>
|
||||
<classpathentry kind="lib" path="lib/druid-1.0.29.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jfinal-3.4-bin-with-src.jar"/>
|
||||
<classpathentry kind="lib" path="lib/c3p0-0.9.1.2.jar"/>
|
||||
<classpathentry kind="lib" path="lib/mysql-connector-java-5.1.13.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-1.2.16.jar"/>
|
||||
<classpathentry kind="lib" path="lib/quartz-2.2.3.jar" sourcepath="D:/装机软件/quartz-2.2.3-distribution/src"/>
|
||||
<classpathentry kind="lib" path="lib/quartz-jobs-2.2.3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-api-1.7.7.jar"/>
|
||||
<classpathentry kind="lib" path="lib/slf4j-log4j12-1.7.7.jar"/>
|
||||
<classpathentry kind="lib" path="lib/mysql-connector-java-5.1.36.jar"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#\u4e3b\u5e93\u540c\u6b65\u5206\u5e93\u6570\u636e\u7ebf\u7a0b\u65f6\u95f4\u5dee
|
||||
syncMaterToSlaveTime=30000
|
||||
syncMaterToSlaveTime=10
|
||||
#\u4e3b\u5e93\u540c\u6b65\u5206\u5e93\u6570\u636e\u7ebf\u7a0b\u65f6\u95f4\u5dee
|
||||
syncSlaveToMaterTime=60000
|
||||
syncSlaveToMaterTime=15
|
||||
#从库向主库同步的线程池数量
|
||||
slave.to.master.pool.num=3
|
||||
#主库向从库同步的线程池数量
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
log4j.rootLogger=DEBUG, stdout, file
|
||||
log4j.rootLogger=INFO, stdout, file
|
||||
#log4j.rootLogger=ERROR, stdout, file
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=%n%-d{yyyy-MM-dd HH:mm:ss}%n[%p]-[Thread: %t]-[%C.%M()]: %m%n
|
||||
log4j.appender.stdout.layout.ConversionPattern=%d %p (%F:%L) [%t] - <%m>%n
|
||||
|
||||
log4j.logger.com.nms=debug
|
||||
# Output to the File
|
||||
#log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
|
||||
#log4j.appender.file.DatePattern='_'yyyy-MM-dd'.log'
|
||||
|
||||
28
nms_sync/conf/quartz.properties
Normal file
28
nms_sync/conf/quartz.properties
Normal file
@@ -0,0 +1,28 @@
|
||||
#################### quartz 相关配置文件 #########################
|
||||
|
||||
#Main Scheduler Settings
|
||||
org.quartz.scheduler.instanceName=NMSSync
|
||||
org.quartz.scheduler.instanceId=AUTO
|
||||
org.quartz.scheduler.threadsInheritContextClassLoaderOfInitializer=true
|
||||
org.quartz.scheduler.skipUpdateCheck=true
|
||||
org.quartz.scheduler.batchTriggerAcquisitionMaxCount=100
|
||||
org.quartz.threadPool.threadCount=10
|
||||
|
||||
#Configure JDBC-JobStoreTX
|
||||
org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX
|
||||
org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate
|
||||
org.quartz.jobStore.tablePrefix=QRTZ_
|
||||
org.quartz.jobStore.isClustered=true
|
||||
org.quartz.jobStore.acquireTriggersWithinLock=true
|
||||
org.quartz.jobStore.clusterCheckinInterval = 30000
|
||||
org.quartz.jobStore.dataSource=qzDS
|
||||
|
||||
#============================================================================
|
||||
# Configure Datasources
|
||||
#============================================================================
|
||||
#JDBC驱动
|
||||
org.quartz.dataSource.qzDS.driver=com.mysql.jdbc.Driver
|
||||
#在程序里赋值
|
||||
#org.quartz.dataSource.qzDS.URL=jdbc:mysql://localhost:3306/quartz
|
||||
#org.quartz.dataSource.qzDS.user=root
|
||||
#org.quartz.dataSource.qzDS.password=christmas258@
|
||||
Binary file not shown.
BIN
nms_sync/lib/log4j-1.2.16.jar
Normal file
BIN
nms_sync/lib/log4j-1.2.16.jar
Normal file
Binary file not shown.
Binary file not shown.
BIN
nms_sync/lib/mysql-connector-java-5.1.36.jar
Normal file
BIN
nms_sync/lib/mysql-connector-java-5.1.36.jar
Normal file
Binary file not shown.
BIN
nms_sync/lib/quartz-2.2.3.jar
Normal file
BIN
nms_sync/lib/quartz-2.2.3.jar
Normal file
Binary file not shown.
BIN
nms_sync/lib/quartz-jobs-2.2.3.jar
Normal file
BIN
nms_sync/lib/quartz-jobs-2.2.3.jar
Normal file
Binary file not shown.
BIN
nms_sync/lib/slf4j-api-1.7.7.jar
Normal file
BIN
nms_sync/lib/slf4j-api-1.7.7.jar
Normal file
Binary file not shown.
BIN
nms_sync/lib/slf4j-log4j12-1.7.7.jar
Normal file
BIN
nms_sync/lib/slf4j-log4j12-1.7.7.jar
Normal file
Binary file not shown.
8346
nms_sync/sql/nms-master.sql
Normal file
8346
nms_sync/sql/nms-master.sql
Normal file
File diff suppressed because it is too large
Load Diff
6452
nms_sync/sql/nms-slave1.sql
Normal file
6452
nms_sync/sql/nms-slave1.sql
Normal file
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@ import org.apache.log4j.Logger;
|
||||
import com.jfinal.plugin.activerecord.ICallback;
|
||||
|
||||
public class SyncStoredProcedure implements ICallback{
|
||||
private Logger logger=Logger.getLogger(this.getClass());
|
||||
private static Logger logger=Logger.getLogger(SyncStoredProcedure.class);
|
||||
private String proName;
|
||||
private String tableName;
|
||||
private String filedAndType;
|
||||
@@ -27,25 +27,12 @@ public class SyncStoredProcedure implements ICallback{
|
||||
public Object call(Connection conn) throws SQLException {
|
||||
logger.info("开始调用存储过程任务");
|
||||
CallableStatement proc=null;
|
||||
try{
|
||||
proc=conn.prepareCall("{call "+proName+"(?,?,?)}");
|
||||
proc.setString(1,tableName);
|
||||
proc.setString(2,filedAndType);
|
||||
proc.setString(3, fileds);
|
||||
proc.execute();
|
||||
logger.info("调用存储过程任务结束");
|
||||
} catch (Exception e){
|
||||
logger.error("调用存储过程任务出现错误 存储过程名称"+proName+" 表名"+tableName+"参数 "+filedAndType+"------"+fileds);
|
||||
logger.error("调用存储过程任务出现错误 ",e);
|
||||
e.printStackTrace();
|
||||
} finally{
|
||||
if(conn!=null){
|
||||
conn.close();
|
||||
}
|
||||
if(proc!=null){
|
||||
proc.close();
|
||||
}
|
||||
}
|
||||
proc=conn.prepareCall("{call "+proName+"(?,?,?)}");
|
||||
proc.setString(1,tableName);
|
||||
proc.setString(2,filedAndType);
|
||||
proc.setString(3, fileds);
|
||||
proc.execute();
|
||||
logger.info("调用存储过程任务结束");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
198
nms_sync/src/com/nms/job/CheckTypeInfoInterceptor.java
Normal file
198
nms_sync/src/com/nms/job/CheckTypeInfoInterceptor.java
Normal file
@@ -0,0 +1,198 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.ICallback;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.interceptor.SyncStoredProcedure;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.SyncCommon;
|
||||
/**
|
||||
* check_type_info
|
||||
* 主库向从库同步 update
|
||||
* 1、update 更新操作 判断 state == 0，监测 表是否已经创建，如果没有创建，新增表
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class CheckTypeInfoInterceptor implements Interceptor{
|
||||
private static final Logger logger = Logger.getLogger(CheckTypeInfoInterceptor.class);
|
||||
|
||||
private TableSyncInfo tableInfo;
|
||||
private SyncDbInfo from;
|
||||
private SyncDbInfo to;
|
||||
public void init(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
this.tableInfo = tableInfo;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exce(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
final String idName = tableInfo.getIdName();
|
||||
final String tableName = tableInfo.getTableName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
Integer event = tableInfo.getEvent();
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
if(event == 1) {//insert，metadata表主键生成方式为 mode = 1,insert 不需要删除 主键id
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, findRecord, batchSize);//添加新增
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else if (event == 2) {//update
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
boolean updateIdName = idName.equalsIgnoreCase(updateIds);//更新主键 是否和 查询主键一致
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
createTable(findRecord, tableInfo, from, to);;
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}else {
|
||||
logger.warn(String.format("unknown event %s", event));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查创建表
|
||||
*/
|
||||
public void createTable(List<Record> findRecord,TableSyncInfo tableInfo, SyncDbInfo masterDb, SyncDbInfo slaveDb) {
|
||||
for(Record record : findRecord) {
|
||||
String creteState = record.get("CRETE_STATE");
|
||||
String tableName = record.getStr("TABLE_NAME");
|
||||
if("0".equals(creteState)) {//表创建
|
||||
Record isExist = Db.use(slaveDb.getDbKey()).findFirst("select 1 count from information_schema.`TABLES` WHERE TABLE_SCHEMA = ? and UPPER(TABLE_NAME) = ?",slaveDb.getDatabaseName(),tableName);
|
||||
if(isExist == null) {
|
||||
//创建表数据
|
||||
StringBuffer sqlSb= new StringBuffer();
|
||||
StringBuffer cIndexFileds = new StringBuffer("data_check_time:seq_id:detection_set_info_id:");
|
||||
List<Record> metadatas = Db.use(slaveDb.getDbKey()).find("select * from metadata where 1=1 and check_type_id=? and state = '0' order by show_num asc",record.getLong("ID"));
|
||||
if(metadatas!=null && metadatas.size()>0) {
|
||||
for(Record meta : metadatas) {
|
||||
sqlSb.append(",").append(meta.get("filed_name")).append(" ").append(toMysqlType(meta.getStr("filed_type")));
|
||||
if(meta.get("chart_state").equals("0")) {
|
||||
cIndexFileds.append(meta.getStr("filed_name")).append(":");
|
||||
}
|
||||
}
|
||||
sqlSb.deleteCharAt(0);
|
||||
//调用存储过程创建表
|
||||
Db.use(slaveDb.getDbKey()).execute(new SyncStoredProcedure("pro_createTable",tableName, sqlSb.toString(),cIndexFileds.toString()));
|
||||
Record syncInfo = new Record();
|
||||
record.set("table_name", tableName);
|
||||
record.set("mode", 2);
|
||||
record.set("id_name", "ID");
|
||||
record.set("event",1);
|
||||
record.set("last_id", 0);
|
||||
record.set("db_id", masterDb.getId());//监测数据，分库向主库同步
|
||||
record.set("last_date", new Date());
|
||||
Db.use(slaveDb.getDbKey()).save("table_sync_info", syncInfo);//添加同步信息
|
||||
logger.info("");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private static String toMysqlType(String type){
|
||||
type = type.trim().toLowerCase();
|
||||
if(type.startsWith("date")){
|
||||
type = type.replaceAll("date", "datetime");
|
||||
}else if(type.startsWith("number")){
|
||||
type = type.replaceAll("number", "bigint");
|
||||
}else if(type.startsWith("varchar")){
|
||||
type = type.replaceAll("varchar2", "varchar");
|
||||
}
|
||||
return type;
|
||||
}
|
||||
|
||||
}
|
||||
248
nms_sync/src/com/nms/job/CommonMissionInterceptor.java
Normal file
248
nms_sync/src/com/nms/job/CommonMissionInterceptor.java
Normal file
@@ -0,0 +1,248 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.SyncCommon;
|
||||
|
||||
/**
|
||||
* 普通任务 mission_result_table146 特殊处理
|
||||
* insert & update
|
||||
* 从库 向 主库 同步，更新 mission_state_table
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class CommonMissionInterceptor implements Interceptor {
|
||||
private static Logger logger = Logger.getLogger(CommonMissionInterceptor.class);
|
||||
private SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //Java Date 类型数据格式化格式
|
||||
// 线程变量记录周期任务结果修改的任务id
|
||||
private Set<Long> updateMissionIds = new HashSet<Long>();
|
||||
// 线程变量记录周期任务结果新增的任务id
|
||||
private Set<Long> insertMissionIds = new HashSet<Long>();
|
||||
|
||||
private TableSyncInfo tableInfo;
|
||||
private SyncDbInfo from;
|
||||
private SyncDbInfo to;
|
||||
public void init(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
this.tableInfo = tableInfo;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exce(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
final String idName = tableInfo.getIdName();
|
||||
final String tableName = tableInfo.getTableName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
Integer event = tableInfo.getEvent();
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
if(event == 1) {//insert
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
logger.debug(String.format("sql -> %s", sbSql.toString()));
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
final List<Record> insertList = new ArrayList<Record>();
|
||||
final List<Record> updateList = new ArrayList<Record>();
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
Record exist = toDbPro.findFirst(String.format("select 1 from %s where %s = ? ", tableName,idName),r.get(idName));
|
||||
if(exist == null) {//不存在
|
||||
insertList.add(r);
|
||||
insertMissionIds.add(r.getLong(idName));
|
||||
}else {
|
||||
updateList.add(r);
|
||||
updateMissionIds.add(r.getLong(idName));
|
||||
}
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, insertList, batchSize);//添加新增
|
||||
toDbPro.batchUpdate(tableName, idName, updateList, batchSize);//更新
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else if (event == 2) {//update
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
boolean updateIdName = idName.equalsIgnoreCase(updateIds);//更新主键 是否和 查询主键一致
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
updateMissionIds.add(r.getLong(idName));
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}else {
|
||||
logger.warn(String.format("unknown event %s", event));
|
||||
}
|
||||
//更新 mission_state_table 表的状态
|
||||
updateMissionStateTableLoop();
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步 loop mission result 表 之后 更新 mission_state_table 表的状态
|
||||
*/
|
||||
private void updateMissionStateTableLoop() {
|
||||
for(Long id :insertMissionIds) {
|
||||
statisticalHandle(id, true);
|
||||
}
|
||||
for(Long id :updateMissionIds) {
|
||||
statisticalHandle(id, false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理非周期任务的统计功能
|
||||
*/
|
||||
public void statisticalHandle(Long missionId,boolean isInsert) {
|
||||
logger.info("根据当前任务id为:"+missionId+"开始统计");
|
||||
// mission_state_table 状态值
|
||||
Integer status = null;
|
||||
List<Record> results = Db.use().find("select result from (\r\n" +
|
||||
"(select result,mission_id from mission_result_table1 mrt) union all \r\n" +
|
||||
"(select result,mission_id from mission_result_table4 mrt) union all \r\n" +
|
||||
"(select result,mission_id from mission_result_table6 mrt)\r\n" +
|
||||
") t \r\n" +
|
||||
"left join mission_state_table mst on mst.mission_id = t.mission_id \r\n" +
|
||||
"where mst.is_loop = 0 and mst.mission_id=?",missionId);
|
||||
|
||||
// 判断任务结果有没有状态值为3的 如果有 则任务状态为在下发
|
||||
Record result = Db.use().findFirst("select t.mission_id,t.ok,t.fail,t.total from (\r\n" +
|
||||
"(select mrt.mission_id,sum(CASE mrt.result when 0 THEN 1 ELSE 0 end) ok,sum(CASE mrt.result when 1 then 1 when -1 then 1 else 0 end) fail,count(mrt.seq_id) total from mission_result_table1 mrt group by mrt.mission_id) union all \r\n" +
|
||||
"(select mrt.mission_id,sum(CASE mrt.result when 0 THEN 1 ELSE 0 end) ok,sum(CASE mrt.result when 1 then 1 when -1 then 1 else 0 end) fail,count(mrt.seq_id) total from mission_result_table4 mrt group by mrt.mission_id) union all \r\n" +
|
||||
"(select mrt.mission_id,sum(CASE mrt.result when 0 THEN 1 ELSE 0 end) ok,sum(CASE mrt.result when 1 then 1 when -1 then 1 else 0 end) fail,count(mrt.seq_id) total from mission_result_table6 mrt group by mrt.mission_id)\r\n" +
|
||||
") t \r\n" +
|
||||
"left join mission_state_table mst on mst.mission_id = t.mission_id \r\n" +
|
||||
"where mst.is_loop = 0 and t.mission_id = ?",missionId);
|
||||
if(null!=result) {
|
||||
Integer okCount = result.getInt("ok");
|
||||
Integer failCount =result.getInt("fail");
|
||||
Integer total =result.getInt("total");
|
||||
|
||||
|
||||
if(okCount+failCount==total) {
|
||||
if(failCount==0) {
|
||||
status=30;
|
||||
}else if(okCount==0) {
|
||||
status=31;
|
||||
}else {
|
||||
status=32;
|
||||
}
|
||||
}else{
|
||||
status=2;
|
||||
}
|
||||
|
||||
logger.info("统计完成 修改mission_state_table状态为:"+status+" 总个数:"+total+"执行数:"+(okCount+failCount)+" 成功:"+okCount+" 失败:"+failCount+"");
|
||||
|
||||
String missionStateAutoDesc=null;
|
||||
if(isInsert) {
|
||||
Record missionStateTable = Db.use().findFirst("select * from mission_state_table where mission_id = ?",missionId);
|
||||
if(null!=missionStateTable&&null!=missionStateTable.get("auto_desc")&&missionStateTable.getStr("auto_desc").length()>0) {
|
||||
return;
|
||||
}
|
||||
missionStateAutoDesc=format.format(System.currentTimeMillis())+"i18n_server.MissionConstants.NOTICE_TASK_RUNNING_n81i </br>";
|
||||
}
|
||||
if(okCount+failCount==total) {
|
||||
missionStateAutoDesc=format.format(System.currentTimeMillis())+" i18n_sserver.UpgradeService.sql.complate_n81i "+total+" i18n_sserver.UpgradeService.sql.executeNode_n81i "+okCount+" i18n_sserver.UpgradeService.sql.failed_n81i "+failCount;
|
||||
}
|
||||
String missionStateDesc="\r\n" +
|
||||
"i18n_server.UpgradeService.sql.total_n81i "+total+" i18n_server.UpgradeService.sql.executeNode2_n81i，</br> "+(total-okCount-failCount)+" i18n_server.UpgradeService.sql.unexecute_n81i，</br>"+(okCount+failCount)+" i18n_server.UpgradeService.sql.execute_n81i【i18n_server.UpgradeService.sql.success_n81i "+okCount+" i18n_sserver.UpgradeService.sql.failed_n81i "+failCount+"】";
|
||||
|
||||
Record missionStateTableResult =new Record();
|
||||
missionStateTableResult.set("mission_id", missionId);
|
||||
if(status!=null) {
|
||||
missionStateTableResult.set("mission_state", status);
|
||||
}
|
||||
missionStateTableResult.set("mission_state_desc", missionStateDesc);
|
||||
if(missionStateAutoDesc!=null) {
|
||||
missionStateTableResult.set("auto_desc", missionStateAutoDesc);
|
||||
}
|
||||
//to 为主库
|
||||
Db.use(to.getDbKey()).update("mission_state_table","mission_id",missionStateTableResult);
|
||||
}
|
||||
logger.info("修改mission_state_table信息完成");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
238
nms_sync/src/com/nms/job/DetectionSetInfoInterceptor.java
Normal file
238
nms_sync/src/com/nms/job/DetectionSetInfoInterceptor.java
Normal file
@@ -0,0 +1,238 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.jfinal.kit.PropKit;
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SetInfo;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.socket.SocketClientServeice;
|
||||
import com.nms.util.SyncCommon;
|
||||
|
||||
/**
|
||||
* detection_set_info 特殊处理
|
||||
* insert & update
|
||||
* 1、主库向 分库同步
|
||||
* 2、主库同步完成后， socket 通知 dc
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class DetectionSetInfoInterceptor implements Interceptor {
|
||||
private static Logger logger = Logger.getLogger(DetectionSetInfoInterceptor.class);
|
||||
private SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //Java Date 类型数据格式化格式
|
||||
|
||||
/**
|
||||
* 监控设置信息变更请求命令
|
||||
*/
|
||||
private static final String WEB_NOTICE_SET_INFO_ALERT = "char:setInfoAlert";
|
||||
|
||||
private TableSyncInfo tableInfo;
|
||||
private SyncDbInfo from;
|
||||
private SyncDbInfo to;
|
||||
public void init(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
this.tableInfo = tableInfo;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exce(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
final String idName = tableInfo.getIdName();
|
||||
final String tableName = tableInfo.getTableName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
Integer event = tableInfo.getEvent();
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
if(event == 1) {//insert
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
Set<Long> oldSetInfo = new HashSet<Long>();
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
StringBuilder wSb = new StringBuilder();
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
oldSetInfo.add(r.getLong(idName));
|
||||
wSb.append(",").append("?");
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
wSb.deleteCharAt(0);
|
||||
boolean tx = fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
if(tx) {
|
||||
List<Record> newSetInfoList = Db.use(to.getDbKey()).find("select m.*,cti.id CHECK_TYPE_INFO_ID,cti.check_type_name CHECK_TYPE_INFO_NAME from detection_set_info m left join check_type_info cti on m.check_type_id=cti.id where m.id in ("+wSb.toString()+")",oldSetInfo.toArray());
|
||||
noticDcSetInfo(newSetInfoList, null, tableInfo, from, to);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else if (event == 2) {//update
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
boolean updateIdName = idName.equalsIgnoreCase(updateIds);//更新主键 是否和 查询主键一致
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
//保存 监测设置 修改之前的记录
|
||||
Set<Long> oldSetInfo = new HashSet<Long>();
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
StringBuilder wSb = new StringBuilder();
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
oldSetInfo.add(r.getLong(idName));
|
||||
wSb.append(",").append("?");
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
wSb.deleteCharAt(0);
|
||||
List<Record> oldSetInfoList = Db.use(to.getDbKey()).find("select m.*,cti.id CHECK_TYPE_INFO_ID,cti.check_type_name CHECK_TYPE_INFO_NAME from detection_set_info m left join check_type_info cti on m.check_type_id=cti.id where m.id in ("+wSb.toString()+")",oldSetInfo.toArray());
|
||||
boolean tx = fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
if(tx) {
|
||||
List<Record> newSetInfoList = Db.use(to.getDbKey()).find("select m.*,cti.id CHECK_TYPE_INFO_ID,cti.check_type_name CHECK_TYPE_INFO_NAME from detection_set_info m left join check_type_info cti on m.check_type_id=cti.id where m.id in ("+wSb.toString()+")",oldSetInfo.toArray());
|
||||
noticDcSetInfo(newSetInfoList, oldSetInfoList, tableInfo, from, to);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}else {
|
||||
logger.warn(String.format("unknown event %s", event));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void noticDcSetInfo(List<Record> newSetInfo,List<Record> oldSetInfo,TableSyncInfo tableInfo, SyncDbInfo masterDb, final SyncDbInfo slaveDb) {
|
||||
Map<Long,Record> oldMap = new HashMap<Long,Record>();
|
||||
if(oldSetInfo != null) {
|
||||
for(Record r : oldSetInfo) {
|
||||
oldMap.put(r.getLong("ID"), r);
|
||||
}
|
||||
}
|
||||
//获取当前分库连接的dc的IP信息
|
||||
final String serverIp = PropKit.use("socket.properties").get("db"+slaveDb.getIp());
|
||||
logger.info("获取socket连接ip信息为："+serverIp);
|
||||
//根据数据库分库dc ip信息获取数据库相关实体信息
|
||||
final Record serverTableInfo = Db.findFirst("select * from server_table where server_ip = ? and server_state=0 ",serverIp);
|
||||
|
||||
for(Record newRecord : newSetInfo) {
|
||||
Record oldRecord = oldMap.get(newRecord.getLong("ID"));
|
||||
SetInfo o=null;
|
||||
if(oldRecord != null) {
|
||||
o = new SetInfo();
|
||||
o.setId(oldRecord.getLong("ID"));
|
||||
o.setCheckTypeId(oldRecord.getLong("CHECK_TYPE_INFO_ID"));
|
||||
o.setCheckTypeName(oldRecord.getStr("CHECK_TYPE_INFO_Name"));
|
||||
o.setProcessIden(oldRecord.getStr("PROCESS_IDEN"));
|
||||
o.setNodeGroupsId(oldRecord.getStr("NODE_GROUPS_ID"));
|
||||
o.setNodeIpsId(oldRecord.getStr("NODE_IPS_ID"));
|
||||
o.setCheckWay(oldRecord.getStr("CHECK_WAY"));
|
||||
}
|
||||
SetInfo n = new SetInfo();
|
||||
n.setId(newRecord.getLong("ID"));
|
||||
n.setCheckTypeId(newRecord.getLong("CHECK_TYPE_INFO_ID"));
|
||||
n.setCheckTypeName(newRecord.getStr("CHECK_TYPE_INFO_Name"));
|
||||
n.setProcessIden(newRecord.getStr("PROCESS_IDEN"));
|
||||
n.setNodeGroupsId(newRecord.getStr("NODE_GROUPS_ID"));
|
||||
n.setNodeIpsId(newRecord.getStr("NODE_IPS_ID"));
|
||||
n.setCheckWay(newRecord.getStr("CHECK_WAY"));
|
||||
final JSONObject jObject = new JSONObject();
|
||||
jObject.put("old", o);
|
||||
jObject.put("new", n);
|
||||
new Thread(new Runnable(){
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
new SocketClientServeice(serverIp).sendInfoToServer(WEB_NOTICE_SET_INFO_ALERT,jObject.toString());
|
||||
logger.info(String.format("监测设置变更通知成功，data: %s", jObject.toString()));
|
||||
} catch (Exception e) {
|
||||
Record eventRecordLibrary=new Record();
|
||||
eventRecordLibrary.set("record_command", WEB_NOTICE_SET_INFO_ALERT);
|
||||
eventRecordLibrary.set("record_content", jObject.toString());
|
||||
eventRecordLibrary.set("record_type", "W2S");
|
||||
eventRecordLibrary.set("state", 1l);
|
||||
eventRecordLibrary.set("nmsserver_id", serverTableInfo.get("id"));
|
||||
eventRecordLibrary.set("create_time", new Date());
|
||||
Db.use(slaveDb.getDbKey()).save("event_record_library", eventRecordLibrary);
|
||||
logger.warn("Monitoring setting to change communication anomalies,",e);
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
13
nms_sync/src/com/nms/job/Interceptor.java
Normal file
13
nms_sync/src/com/nms/job/Interceptor.java
Normal file
@@ -0,0 +1,13 @@
|
||||
package com.nms.job;
|
||||
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
|
||||
/**
|
||||
* 名字叫拦截器，但功能和拦截器不同，如果配置了拦截器则只调用此 exce 方法，不在执行下面的 insert update 等
|
||||
* @author ThinkPad
|
||||
*
|
||||
*/
|
||||
public interface Interceptor {
|
||||
void exce(final TableSyncInfo tableInfo,final SyncDbInfo from ,final SyncDbInfo to);
|
||||
}
|
||||
344
nms_sync/src/com/nms/job/LoopStateTableInterceptor.java
Normal file
344
nms_sync/src/com/nms/job/LoopStateTableInterceptor.java
Normal file
@@ -0,0 +1,344 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.SyncCommon;
|
||||
|
||||
/**
|
||||
* loopmission_state_table 特殊处理
|
||||
* insert & update
|
||||
* 1、分库向主库同步
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class LoopStateTableInterceptor implements Interceptor {
|
||||
private static Logger logger = Logger.getLogger(LoopStateTableInterceptor.class);
|
||||
private SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //Java Date 类型数据格式化格式
|
||||
|
||||
// 线程变量记录周期任务结果修改的任务id
|
||||
private Set<Long> loopUpdateMissionIds = new HashSet<Long>();
|
||||
// 线程变量记录周期任务结果新增的任务id
|
||||
private Set<Long> loopInsertMissionIds = new HashSet<Long>();
|
||||
|
||||
private TableSyncInfo tableInfo;
|
||||
private SyncDbInfo from;
|
||||
private SyncDbInfo to;
|
||||
public void init(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
this.tableInfo = tableInfo;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exce(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
final String idName = tableInfo.getIdName();
|
||||
final String tableName = tableInfo.getTableName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
Integer event = tableInfo.getEvent();
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
if(event == 1) {//insert
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
final List<Record> insertList = new ArrayList<Record>();
|
||||
final List<Record> updateList = new ArrayList<Record>();
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
Record exist = toDbPro.findFirst(String.format("select 1 from %s where %s = ? ", tableName,idName),r.get(idName));
|
||||
if(exist == null) {//不存在
|
||||
insertList.add(r);
|
||||
loopInsertMissionIds.add(r.getLong(idName));
|
||||
}else {
|
||||
updateList.add(r);
|
||||
loopUpdateMissionIds.add(r.getLong(idName));
|
||||
}
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, insertList, batchSize);//添加新增
|
||||
toDbPro.batchUpdate(tableName, idName, updateList, batchSize);//更新
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else if (event == 2) {//update
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
boolean updateIdName = idName.equalsIgnoreCase(updateIds);//更新主键 是否和 查询主键一致
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
loopUpdateMissionIds.add(r.getLong(idName));
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}else {
|
||||
logger.warn(String.format("unknown event %s", event));
|
||||
}
|
||||
//更新 mission_state_table 表的状态
|
||||
updateMissionStateTableLoop();
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步 loop mission result 表 之后 更新 mission_state_table 表的状态
|
||||
*/
|
||||
private void updateMissionStateTableLoop() {
|
||||
for(Long id :loopInsertMissionIds) {
|
||||
statisticalLoopHandle(id, true);
|
||||
}
|
||||
for(Long id :loopUpdateMissionIds) {
|
||||
statisticalLoopHandle(id, false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 处理周期任务的统计功能
|
||||
*/
|
||||
public void statisticalLoopHandle(Long curMissionId,boolean isInsert) {
|
||||
Record loopmissionStateTableInfo = Db.use().findFirst("select * from loopmission_state_table where cur_mission_id = ? ",curMissionId);
|
||||
String missionDesc="";
|
||||
Integer status=null;
|
||||
String autoDesc="";
|
||||
Record missionTableInfo =null;
|
||||
Integer missionState = null;
|
||||
// 判断任务状态为1 根据同步结果数据 修改为状态2 正在执行
|
||||
if(null!=loopmissionStateTableInfo) {
|
||||
missionTableInfo = Db.use().findFirst("select * from mission_state_table where mission_id = ? ",loopmissionStateTableInfo.getLong("mission_id"));
|
||||
if(null!=missionTableInfo&&missionTableInfo.getInt("mission_state").equals(1)&&isInsert) {
|
||||
missionTableInfo.set("mission_state",2);
|
||||
if(null==missionTableInfo.getStr("auto_desc")) {
|
||||
missionTableInfo.set("auto_desc", format.format(System.currentTimeMillis())+"i18n_server.LoadNewMissionThread.missionStart_n81i");
|
||||
}
|
||||
//更新 主库 mission_state_table 表 任务执行描述
|
||||
Db.use(to.getDbKey()).update("mission_state_table", missionTableInfo);
|
||||
}
|
||||
|
||||
// 修改周期任务的执行状态
|
||||
Record result = null;
|
||||
// 周期任务时进行统计
|
||||
if(loopmissionStateTableInfo.getInt("mission_state").equals(7)) {
|
||||
result=Db.use().findFirst("select lmst.mission_id missionId, t.ok, t.fail, t.total\r\n" +
|
||||
"from (select mrt.mission_id, \r\n" +
|
||||
"ifnull(sum(case mrt.result when 7 then 1 else 0 end), 0) ok, \r\n" +
|
||||
"ifnull(sum(case mrt.result when 4 then 1 else 0 end), 0) fail, \r\n" +
|
||||
"count(mrt.seq_id) total \r\n" +
|
||||
"from mission_result_table4 mrt \r\n" +
|
||||
"group by mrt.mission_id) t \r\n" +
|
||||
"left join (select max(lst.cur_mission_id) cur_mission_id, lst.mission_id \r\n" +
|
||||
"from mission_result_table4 mrt4 \r\n" +
|
||||
"left join loopmission_state_table lst on mrt4.mission_id = lst.cur_mission_id \r\n" +
|
||||
"group by lst.mission_id) lmst \r\n" +
|
||||
"on lmst.cur_mission_id = t.mission_id \r\n" +
|
||||
"left join mission_state_table mst \r\n" +
|
||||
"on mst.mission_id = lmst.mission_id \r\n" +
|
||||
"where lmst.cur_mission_id is not null and mst.is_loop=1 \r\n" +
|
||||
"and mst.mission_id = ? ", loopmissionStateTableInfo.getLong("mission_id"));
|
||||
}else {
|
||||
result=Db.use().findFirst("select lmst.mission_id missionId, t.ok, t.fail, t.total\r\n" +
|
||||
"from (select mrt.mission_id, \r\n" +
|
||||
"ifnull(sum(case mrt.result when 0 then 1 else 0 end), 0) ok, \r\n" +
|
||||
"ifnull(sum(case mrt.result when 1 then 1 when -1 then 1 else 0 end), 0) fail, \r\n" +
|
||||
"count(mrt.seq_id) total \r\n" +
|
||||
"from mission_result_table4 mrt \r\n" +
|
||||
"group by mrt.mission_id) t \r\n" +
|
||||
"left join (select max(lst.cur_mission_id) cur_mission_id, lst.mission_id \r\n" +
|
||||
"from mission_result_table4 mrt4 \r\n" +
|
||||
"left join loopmission_state_table lst on mrt4.mission_id = lst.cur_mission_id \r\n" +
|
||||
"group by lst.mission_id) lmst \r\n" +
|
||||
"on lmst.cur_mission_id = t.mission_id \r\n" +
|
||||
"left join mission_state_table mst \r\n" +
|
||||
"on mst.mission_id = lmst.mission_id \r\n" +
|
||||
"where lmst.cur_mission_id is not null and mst.is_loop=1 \r\n" +
|
||||
"and mst.mission_id = ? ", loopmissionStateTableInfo.getLong("mission_id"));
|
||||
}
|
||||
if(null!=result) {
|
||||
if(null!=result.get("missionId")&&result.getStr("missionId").length()>0) {
|
||||
Long ok = result.getLong("ok");
|
||||
Long fail =result.getLong("fail");
|
||||
Long total = result.getLong("total");
|
||||
missionDesc="i18n_server.UpgradeService.sql.cycle_n81i "+total+" i18n_server.UpgradeService.sql.executeNode2_n81i，</br>"+(total-ok-fail)+" i18n_server.UpgradeService.sql.unexecute_n81i，</br>"+(ok+fail)+" i18n_server.UpgradeService.sql.execute_n81i【i18n_server.UpgradeService.sql.success_n81i "+ok+" i18n_sserver.UpgradeService.sql.failed_n81i "+fail+" 】";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(isInsert) {
|
||||
missionState = loopmissionStateTableInfo.getInt("mission_state");
|
||||
Integer missionTableInfoState = missionTableInfo.getInt("mission_state");
|
||||
switch(missionState) {
|
||||
case(3):
|
||||
Record findFirst = Db.use().findFirst("select COUNT(*) count from loopmission_state_table where mission_id = ? and mission_state=0",loopmissionStateTableInfo.getLong("mission_id"));
|
||||
if(findFirst.getInt("count").equals(0)&&missionTableInfoState!=3) {
|
||||
status=3;
|
||||
autoDesc=format.format(System.currentTimeMillis())+"i18n_server.UpgradeService.sql.revoke_n81i";
|
||||
}
|
||||
break;
|
||||
case(4):
|
||||
if(missionTableInfoState!=4) {
|
||||
status=4;
|
||||
autoDesc=format.format(System.currentTimeMillis())+" Task execution failure ";
|
||||
}
|
||||
break;
|
||||
case(6):
|
||||
if(missionTableInfoState!=6) {
|
||||
status=6;
|
||||
autoDesc=format.format(System.currentTimeMillis())+"Task revocation start execute";
|
||||
}
|
||||
break;
|
||||
case(7):
|
||||
if(missionTableInfoState!=7) {
|
||||
status=7;
|
||||
autoDesc=format.format(System.currentTimeMillis())+"i18n_server.UpgradeService.sql.revoke_n81i";
|
||||
}
|
||||
break;
|
||||
}
|
||||
}else {
|
||||
missionState = loopmissionStateTableInfo.getInt("mission_state");
|
||||
Integer missionTableInfoState = missionTableInfo.getInt("mission_state");
|
||||
switch(missionState) {
|
||||
case(3):
|
||||
Record findFirst = Db.use().findFirst("select COUNT(*) count from loopmission_state_table where mission_id = ? and mission_state=0",loopmissionStateTableInfo.getLong("mission_id"));
|
||||
if(findFirst.getInt("count").equals(0)&&missionTableInfoState!=3&&missionTableInfoState!=5&&missionTableInfoState!=6&&missionTableInfoState!=7) {
|
||||
status=3;
|
||||
autoDesc=format.format(System.currentTimeMillis())+"i18n_server.UpgradeService.sql.revoke_n81i";
|
||||
}
|
||||
break;
|
||||
case(2):
|
||||
if(missionTableInfoState!=6&&missionTableInfoState!=5&&missionTableInfoState!=7) {
|
||||
status=2;
|
||||
}
|
||||
break;
|
||||
case(6):
|
||||
status=6;
|
||||
if(missionTableInfoState!=6) {
|
||||
autoDesc=format.format(System.currentTimeMillis())+"Task revocation start execute";
|
||||
}
|
||||
break;
|
||||
case(7):
|
||||
Record find = Db.use().findFirst("\r\n" +
|
||||
"\r\n" +
|
||||
"select count(*) count \r\n" +
|
||||
" from (\r\n" +
|
||||
" select lmst.mission_id, \r\n" +
|
||||
" sum(case lmst.mission_state when 3 then 1 when 7 then 1 else 0 end) ok,-- //正常执行完成的周期，撤销完成的周期\r\n" +
|
||||
" sum(case lmst.mission_state when 4 then 1 else 0 end) fail,-- //未能执行的周期\r\n" +
|
||||
" count(lmst.cur_mission_id) total\r\n" +
|
||||
" from loopmission_state_table lmst\r\n" +
|
||||
" group by lmst.mission_id\r\n" +
|
||||
" ) t\r\n" +
|
||||
" left join mission_state_table mst on mst.mission_id = t.mission_id\r\n" +
|
||||
" where t.ok + t.fail = t.total\r\n" +
|
||||
" and ifnull(mst.mission_id, -1) <> -1\r\n" +
|
||||
" and mst.is_loop = 1\r\n" +
|
||||
" and mst.mission_state not in (3, 7, 4) and mst.mission_id=?",missionTableInfo.getLong("mission_id"));
|
||||
if(null!=find&&find.getInt("count")!=0) {
|
||||
status=7;
|
||||
}
|
||||
if(missionTableInfoState!=7) {
|
||||
autoDesc=format.format(System.currentTimeMillis())+"i18n_server.UpgradeService.sql.revoke_n81i";
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Record missionStateTableResult =new Record();
|
||||
missionStateTableResult.set("mission_id", loopmissionStateTableInfo.get("mission_id"));
|
||||
if(status!=null) {
|
||||
missionStateTableResult.set("mission_state", status);
|
||||
}
|
||||
if(missionDesc.length()>0&&null!=missionState) {
|
||||
missionStateTableResult.set("mission_state_desc",missionDesc);
|
||||
}
|
||||
if(autoDesc.length()>0) {
|
||||
missionStateTableResult.set("auto_desc",autoDesc);
|
||||
}
|
||||
//同步更新 主库 mission_state_table 任务执行描述信息
|
||||
Db.use(to.getDbKey()).update("mission_state_table","mission_id",missionStateTableResult);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
177
nms_sync/src/com/nms/job/MetaDataInterceptor.java
Normal file
177
nms_sync/src/com/nms/job/MetaDataInterceptor.java
Normal file
@@ -0,0 +1,177 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.SyncCommon;
|
||||
/**
|
||||
* metadata 表
|
||||
* insert & update
|
||||
* 1、metadata 表insert 时判断是否已经创建了 表，如果没有创建表则不用特殊处理
|
||||
* 2、metadata 表update 不需要更新 表结构
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class MetaDataInterceptor implements Interceptor{
|
||||
private static final Logger logger = Logger.getLogger(MetaDataInterceptor.class);
|
||||
|
||||
private TableSyncInfo tableInfo;
|
||||
private SyncDbInfo from;
|
||||
private SyncDbInfo to;
|
||||
public void init(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
this.tableInfo = tableInfo;
|
||||
this.from = from;
|
||||
this.to = to;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exce(final TableSyncInfo tableInfo, final SyncDbInfo from, final SyncDbInfo to) {
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
init(tableInfo, from, to);//将变量保存到 对象
|
||||
final String idName = tableInfo.getIdName();
|
||||
final String tableName = tableInfo.getTableName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
Integer event = tableInfo.getEvent();
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
if(event == 1) {//insert，metadata表主键生成方式为 mode = 1,insert 不需要删除 主键id
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, findRecord, batchSize);//添加新增
|
||||
for(Record r : findRecord) {
|
||||
tableFiledSync(r.getLong("id"), tableInfo, from, to);
|
||||
}
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else if (event == 2) {//update
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
boolean updateIdName = idName.equalsIgnoreCase(updateIds);//更新主键 是否和 查询主键一致
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}else {
|
||||
logger.warn(String.format("unknown event %s", event));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步 meta_data 表结构字段
|
||||
* @param mdId
|
||||
*/
|
||||
public void tableFiledSync(Long mdId,TableSyncInfo tableInfo, SyncDbInfo masterDb, SyncDbInfo slaveDb) {
|
||||
Record record = Db.use(slaveDb.getDbKey()).findFirst("select m.*,cti.table_name from metadata m left join check_type_info cti on m.check_type_id=cti.id where m.id = ? ",mdId);
|
||||
Record isExist = Db.use(slaveDb.getDbKey()).findFirst("select 1 count from information_schema.`TABLES` WHERE TABLE_SCHEMA = ? and UPPER(TABLE_NAME) = ?",slaveDb.getDatabaseName(),record.get("table_name"));
|
||||
//向数据库中添加新的字段
|
||||
if(isExist != null){
|
||||
StringBuffer sqlString = new StringBuffer("alter table ");
|
||||
sqlString.append(record.getStr("table_name").toUpperCase());
|
||||
sqlString.append(" add(");
|
||||
sqlString.append(record.getStr("filed_name")+" "+ toMysqlType(record.getStr("filed_type"))+")");
|
||||
//执行添加字段
|
||||
int resu =Db.use(slaveDb.getDbKey()).update(sqlString.toString());
|
||||
logger.info(String.format("alter sql - > %s ,result : %s ",sqlString,resu));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String toMysqlType(String type){
|
||||
type = type.trim().toLowerCase();
|
||||
if(type.startsWith("date")){
|
||||
type = type.replaceAll("date", "datetime");
|
||||
}else if(type.startsWith("number")){
|
||||
type = type.replaceAll("number", "bigint");
|
||||
}else if(type.startsWith("varchar")){
|
||||
type = type.replaceAll("varchar2", "varchar");
|
||||
}
|
||||
return type;
|
||||
}
|
||||
|
||||
}
|
||||
416
nms_sync/src/com/nms/job/SyncDataJob.java
Normal file
416
nms_sync/src/com/nms/job/SyncDataJob.java
Normal file
@@ -0,0 +1,416 @@
|
||||
package com.nms.job;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.quartz.DisallowConcurrentExecution;
|
||||
import org.quartz.Job;
|
||||
import org.quartz.JobDataMap;
|
||||
import org.quartz.JobDetail;
|
||||
import org.quartz.JobExecutionContext;
|
||||
import org.quartz.JobExecutionException;
|
||||
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.jfinal.plugin.activerecord.Db;
|
||||
import com.jfinal.plugin.activerecord.DbPro;
|
||||
import com.jfinal.plugin.activerecord.IAtom;
|
||||
import com.jfinal.plugin.activerecord.Record;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.StopWatch;
|
||||
import com.nms.util.SyncCommon;
|
||||
|
||||
/**
|
||||
* 同步数据
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
@DisallowConcurrentExecution
|
||||
public class SyncDataJob implements Job{
|
||||
private static final Logger logger = Logger.getLogger(SyncDataJob.class);
|
||||
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public void execute(JobExecutionContext context) throws JobExecutionException {
|
||||
JobDetail jobDetail = context.getJobDetail();
|
||||
//从jobdatamap 中取到 tablename,fromdb,todb信息
|
||||
JobDataMap jobDataMap = jobDetail.getJobDataMap();
|
||||
String tableName = jobDataMap.getString("tableName");
|
||||
List<SyncDbInfo> fromDb = (List<SyncDbInfo>) jobDataMap.get("fromDb");
|
||||
SyncDbInfo toDb = (SyncDbInfo) jobDataMap.get("toDb");
|
||||
//同步表
|
||||
syncTable(tableName, fromDb, toDb);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 普通通用逻辑的同步表
|
||||
* 处理一对多的情况，防止数据库死锁
|
||||
* 多对一 则直接 拆分为 一对一
|
||||
* @param tableName
|
||||
* @param fromDb
|
||||
* @param toDb
|
||||
*/
|
||||
public void syncTable(String tableName,List<SyncDbInfo> fromDb,SyncDbInfo toDb) {
|
||||
StopWatch sw = new StopWatch();
|
||||
sw.tag("start");
|
||||
for(SyncDbInfo from : fromDb) {
|
||||
sw.tag("from-start");
|
||||
//获取同步表信息 tableSyncInfo
|
||||
String sql = "select * from table_sync_info tsi where tsi.db_id = ? and tsi.table_name = ? order by tsi.event";//按event排序，保证insert,update,delete顺序
|
||||
List<TableSyncInfo> tableSyncInfo = new TableSyncInfo().use(from.getDbKey()).find(sql, toDb.getId(),tableName);//当前表有哪些需要同步
|
||||
if(tableSyncInfo != null &&tableSyncInfo.size() > 0) {
|
||||
boolean allDataSync = false;//是否执行了全量同步
|
||||
TableSyncInfo temp = null;
|
||||
try {
|
||||
for(TableSyncInfo tableInfo : tableSyncInfo) {
|
||||
long tcount = 0;
|
||||
temp = tableInfo;
|
||||
sw.tag("info-start");
|
||||
String interceptor = tableInfo.getInterceptor();
|
||||
if(StrKit.notBlank(interceptor)) {//如果 有 interceptor 直接使用 interceptor 同步，不在执行下面的同步代码
|
||||
Class<?> inCls = Class.forName(interceptor);
|
||||
Interceptor newInstance = (Interceptor)inCls.newInstance();
|
||||
newInstance.exce(tableInfo, from, toDb);
|
||||
continue;
|
||||
}
|
||||
Integer event = tableInfo.getEvent();
|
||||
switch (event) {
|
||||
case 1:
|
||||
//如果 last_id 为 -1：truncate，-2 delete
|
||||
allDataSync = syncAllData(tableName, tableInfo, from, toDb);
|
||||
if(allDataSync) {//执行了全量同步
|
||||
tableInfo.setLastId(0L);
|
||||
}
|
||||
tcount = eventInsert(tableName,tableInfo, from, toDb); break;
|
||||
case 2:
|
||||
if(allDataSync) {//执行了全量同步，再次全量更新
|
||||
tableInfo.setLastId(0L);
|
||||
}
|
||||
tcount = eventUpdate(tableName,tableInfo, from, toDb); break;
|
||||
case 3:
|
||||
if(allDataSync) {//执行了全量同步，再次全量删除
|
||||
tableInfo.setLastId(0L);
|
||||
}
|
||||
tcount = eventDelete(tableName,tableInfo, from, toDb); break;
|
||||
default:
|
||||
logger.warn(String.format("unknown event : %s,tableName: %s", event,tableName));
|
||||
break;
|
||||
}
|
||||
|
||||
sw.tag("info-end");
|
||||
logger.debug(String.format("耗时统计，%s-同步完成，tableName : %s , toDb : %s ,fromDb : %s , total : %s ,耗时：%s", tableInfo.getEventStr(),tableName,toDb.getDbKey(),from.getDbKey(),tcount,sw.timeBetween("info-end", "info-start")));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(String.format("同步异常，tableName : %s,evnet : %s ,from : %s, to : %s ", temp.getTableName(),temp.getEventStr(),from.getDbKey(),toDb.getDbKey()),e);
|
||||
}
|
||||
}
|
||||
sw.tag("from-end");
|
||||
logger.debug(String.format("耗时统计，表同步完成，tableName : %s , toDb : %s ,fromDb : %s , 耗时：%s", tableName,toDb.getDbKey(),from.getDbKey(),sw.timeBetween("from-end", "from-start")));
|
||||
}
|
||||
sw.tag("end");
|
||||
logger.debug(String.format("耗时统计，库同步完成，tableName : %s , toDb : %s ,fromDbSize : %s , 耗时：%s", tableName,toDb.getDbKey(),fromDb.size(),sw.timeBetween("end", "start")));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* inset 同步
|
||||
* @param tableInfo
|
||||
* @param from
|
||||
* @param to
|
||||
* @throws ClassNotFoundException
|
||||
* @throws IllegalAccessException
|
||||
* @throws InstantiationException
|
||||
*/
|
||||
public long eventInsert(final String tableName,final TableSyncInfo tableInfo,SyncDbInfo from ,SyncDbInfo to) throws ClassNotFoundException, InstantiationException, IllegalAccessException {
|
||||
long count = 0;
|
||||
String idName = tableInfo.getIdName();
|
||||
Integer mode = tableInfo.getMode();
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long fromDbId = from.getId();// from db id
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
Long lastId = tableInfo.getLastId();
|
||||
if(StrKit.notBlank(idName)) {//主键自增
|
||||
StringBuilder sbSql = (mode == 2 ? SyncCommon.insertModeTwoSql(tableInfo) : SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo));
|
||||
logger.debug(String.format("sql -> %s", sbSql.toString()));
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
insert:while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
count += findRecord.size();
|
||||
lastId = (mode==2? findRecord.get(findRecord.size()-1).getLong(idName) : findRecord.get(findRecord.size()-1).getLong("tableeventlogid"));
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {
|
||||
//node_table表涉及双向同步，添加同步库 id
|
||||
if("node_table".equalsIgnoreCase(tableName)) {
|
||||
r.set("sync_status",fromDbId);
|
||||
}
|
||||
if("event_record_library".equalsIgnoreCase(tableName)) {
|
||||
//设置同步数据所在数据库的id以及所在原来表的id 用于修改和删除 -1为中心或主库
|
||||
r.set("old_id",r.get(idName));
|
||||
r.set("db_id", fromDbId);
|
||||
r.remove(idName);//删除主键,防止主键冲突
|
||||
}
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
if("detection_info_new".equalsIgnoreCase(tableName)) {
|
||||
//detection_info_new insert 方式，先删除相同 DETECTION_SET_INFO_ID SEQ_ID 的记录，在插入
|
||||
String delColumns = "DETECTION_SET_INFO_ID,SEQ_ID";
|
||||
String delSql = SyncCommon.genDelSql(tableName, delColumns,null);
|
||||
toDbPro.batch(delSql, delColumns, findRecord, batchSize);//删除
|
||||
}
|
||||
toDbPro.batchSave(tableName, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break insert;
|
||||
}
|
||||
}else {
|
||||
//mode=0：需要使用触发器更新 table_event_log 表，同步时需要关联table_event_log 表 mode=1：主键为自定义不重复主键
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
count += findRecord.size();
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
if("event_record_library".equalsIgnoreCase(tableName)) {
|
||||
//设置同步数据所在数据库的id以及所在原来表的id 用于修改和删除 -1为中心或主库
|
||||
r.set("old_id",r.get(idName));
|
||||
r.set("db_id", fromDbId);
|
||||
r.remove(idName);//删除主键,防止主键冲突
|
||||
}
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
r.remove("tableeventlogid");
|
||||
}
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchSave(tableName, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}else {//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* update 同步
|
||||
* @param tableInfo
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public long eventUpdate(final String tableName,final TableSyncInfo tableInfo,SyncDbInfo from ,SyncDbInfo to) {
|
||||
long count = 0;
|
||||
final String idName = tableInfo.getIdName();//查询主键
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
//排除 insert，update 字段
|
||||
String excludeNames = tableInfo.getExcludeNames();
|
||||
String[] excludeNameArr = StrKit.notBlank(excludeNames)? excludeNames.split(",") :null;
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.getSelectSqlLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
count += findRecord.size();
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
if(excludeNameArr != null) {//删除排除字段，不需要同步
|
||||
for(String exclude : excludeNameArr) {
|
||||
r.remove(exclude);
|
||||
}
|
||||
}
|
||||
if("mission_state_table".equalsIgnoreCase(tableName)) {
|
||||
// 主库同步任务状态表 如果状态修改不是周期停用的话 将状态字段的修改过滤
|
||||
if(!(r.getInt("mission_state").equals(5)||r.getInt("mission_state").equals(1))) {
|
||||
r.remove("mission_state");
|
||||
}
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batchUpdate(tableName, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* delete 同步
|
||||
* 1、delete 的where 条件用于更新 目标的 做限制条件
|
||||
* @param tableInfo
|
||||
* @param from
|
||||
* @param to
|
||||
*/
|
||||
public long eventDelete(final String tableName,final TableSyncInfo tableInfo,SyncDbInfo from ,SyncDbInfo to) {
|
||||
long count = 0;
|
||||
final String idName = tableInfo.getIdName();//查询主键
|
||||
final String updateIds = StrKit.notBlank(tableInfo.getUpdateIds())?tableInfo.getUpdateIds():idName ;//更新主键，多个逗号分隔，如果没有配置updateIds ,默认为 idName
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
final Integer batchSize = tableInfo.getBatchSize();
|
||||
String where = tableInfo.getWhere();
|
||||
Long fromDbId = from.getId();
|
||||
if(StrKit.notBlank(idName)) {//单一主键，通过table_event_log表 关联查询
|
||||
StringBuilder sbSql = SyncCommon.delSelectLeftJoinEventLog(tableInfo);
|
||||
Long lastId = tableInfo.getLastId();
|
||||
//生成delete sql 语句
|
||||
final String delSql = SyncCommon.genDelSql(tableName, updateIds, where);
|
||||
while(true) {
|
||||
final List<Record> findRecord = fromDbPro.find(sbSql.toString(), lastId,batchSize);
|
||||
if(findRecord != null && findRecord.size() > 0) {
|
||||
count += findRecord.size();
|
||||
lastId = findRecord.get(findRecord.size()-1).getLong("tableeventlogid");
|
||||
tableInfo.setLastId(lastId);//最后更新的id
|
||||
for(Record r : findRecord) {//删除自增主键，不需要同步
|
||||
r.remove("tableeventlogid");
|
||||
r.set("db_id", fromDbId);//添加 db_id
|
||||
if("event_record_library".equalsIgnoreCase(tableName)) {
|
||||
r.set("old_id",r.getLong(idName));
|
||||
}
|
||||
}
|
||||
fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.batch(delSql, updateIds, findRecord, batchSize);
|
||||
tableInfo.setLastDate(new Date());//最后更新时间
|
||||
fromDbPro.update("table_sync_info", tableInfo.toRecord());
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}else {//没有数据，退出循环
|
||||
break ;
|
||||
}
|
||||
}
|
||||
}else {//多主键的更新，需要每条查询，暂不实现
|
||||
|
||||
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* 全量同步，首先删除表中的内容
|
||||
* event= 1 and last_id = -1 : truncate, last_id = -2 : delete from table where
|
||||
* @param tableName 表名
|
||||
* @param tableInfo 表同步配置信息
|
||||
* @param fromDbPro from数据库信息
|
||||
* @param toDbPro to数据库信息
|
||||
*/
|
||||
public boolean syncAllData(final String tableName,final TableSyncInfo tableInfo,final SyncDbInfo from ,final SyncDbInfo to) {
|
||||
final DbPro fromDbPro = Db.use(from.getDbKey());
|
||||
final DbPro toDbPro = Db.use(to.getDbKey());
|
||||
Long lastId = tableInfo.getLastId();
|
||||
final StringBuilder sql = new StringBuilder();
|
||||
if(lastId == -1) {//trancate 表
|
||||
sql.append("TRUNCATE ").append(tableName);
|
||||
}else if (lastId == -2) {
|
||||
String where = tableInfo.getWhere();
|
||||
sql.append("delete from ").append(tableName);
|
||||
if(StrKit.notBlank(where)) {
|
||||
sql.append(" where ").append(where);
|
||||
}
|
||||
}
|
||||
if(sql.length() > 0) {
|
||||
toDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
return fromDbPro.tx(new IAtom() {
|
||||
@Override
|
||||
public boolean run() throws SQLException {
|
||||
toDbPro.delete(sql.toString());
|
||||
fromDbPro.update("update table_sync_info set last_id = 0 where table_name = ?" ,tableName);
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
192
nms_sync/src/com/nms/main/StartUp.java
Normal file
192
nms_sync/src/com/nms/main/StartUp.java
Normal file
@@ -0,0 +1,192 @@
|
||||
package com.nms.main;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.quartz.Scheduler;
|
||||
import org.quartz.SchedulerException;
|
||||
|
||||
import com.jfinal.kit.Prop;
|
||||
import com.jfinal.kit.PropKit;
|
||||
import com.jfinal.plugin.activerecord.ActiveRecordPlugin;
|
||||
import com.jfinal.plugin.druid.DruidPlugin;
|
||||
import com.nms.job.SyncDataJob;
|
||||
import com.nms.model.SyncDbInfo;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
import com.nms.util.QuartzUtils;
|
||||
/**
|
||||
* 数据同步主功能 相当于主动推送操作
|
||||
* @author Administrator
|
||||
*
|
||||
*/
|
||||
public class StartUp{
|
||||
private static Logger logger = Logger.getLogger(StartUp.class);
|
||||
//druid 配置文件路径
|
||||
public static final String DRUID_CONFIG_FILE_PATH;
|
||||
public static Scheduler SCHEDULER;
|
||||
|
||||
static {
|
||||
URL urlObj = StartUp.class.getClassLoader().getResource("druid.properties");
|
||||
if(urlObj==null){
|
||||
System.err.println("找不到配置文件：druid.properties");
|
||||
logger.error("No configuration file can be found: druid.properties");
|
||||
System.exit(1);
|
||||
}
|
||||
DRUID_CONFIG_FILE_PATH = urlObj.getPath().replaceAll("%20", " ");
|
||||
logger.debug(String.format("druid配置文件路径：", DRUID_CONFIG_FILE_PATH));
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
logger.info("同步程序开始启动");
|
||||
try {
|
||||
//初始化数据源
|
||||
List<SyncDbInfo> dataSource = initDataSource();
|
||||
//初始化 定时器
|
||||
initScheduler();
|
||||
//加载同步表信息，添加定时任务
|
||||
initSyncTable(dataSource);
|
||||
//程序关闭时，停掉 scheduler
|
||||
Runtime.getRuntime().addShutdownHook(new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
SCHEDULER.shutdown();
|
||||
} catch (Exception e) {
|
||||
logger.error("SCHEDULER shutdown error",e);
|
||||
}
|
||||
}
|
||||
});
|
||||
} catch(Exception e) {
|
||||
logger.error("数据同步启动发生异常",e);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 定时器
|
||||
*/
|
||||
private static void initScheduler() {
|
||||
//初始化定时器
|
||||
Properties quartzConfig = PropKit.use("quartz.properties").getProperties();
|
||||
//org.quartz.dataSource.qzDS.URL
|
||||
//org.quartz.dataSource.qzDS.user
|
||||
//org.quartz.dataSource.qzDS.password
|
||||
Prop prop = PropKit.use("db.properties");
|
||||
quartzConfig.put("org.quartz.dataSource.qzDS.URL", prop.get("dburl"));
|
||||
quartzConfig.put("org.quartz.dataSource.qzDS.user", prop.get("dbusername"));
|
||||
quartzConfig.put("org.quartz.dataSource.qzDS.password", prop.get("dbpassword"));
|
||||
try {
|
||||
SCHEDULER = QuartzUtils.createScheduler(quartzConfig);
|
||||
} catch (Exception e) {
|
||||
logger.error("Scheduler init error",e);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 初始化数据源
|
||||
* @return
|
||||
*/
|
||||
public static List<SyncDbInfo> initDataSource() {
|
||||
//从配置文件获取数据库连接信息
|
||||
PropKit.use("db.properties");
|
||||
//创建主数据库数据源
|
||||
DruidPlugin masterDruid=new DruidPlugin(PropKit.get("dburl"),PropKit.get("dbusername"),PropKit.get("dbpassword"));
|
||||
masterDruid.setInitialSize(1);
|
||||
masterDruid.setMaxActive(1);
|
||||
masterDruid.setMinIdle(1);
|
||||
masterDruid.setMaxWait(Integer.valueOf(PropKit.get("dbMaxWait")));
|
||||
ActiveRecordPlugin masterArp=new ActiveRecordPlugin("masterDataSource",masterDruid);
|
||||
masterArp.setShowSql(Boolean.valueOf(PropKit.get("dbShowSql")));
|
||||
masterDruid.start();
|
||||
masterArp.start();
|
||||
logger.info("master datasource connection ok ");
|
||||
List<SyncDbInfo> syncDbInfos = SyncDbInfo.dao.use("masterDataSource").find("select * from sync_db_info");
|
||||
createConn(syncDbInfos);//初始化各个分库
|
||||
logger.info(String.format("数据源初始化完成，datasource total : %s ", syncDbInfos.size()));
|
||||
return syncDbInfos;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化同步信息，创建定时任务
|
||||
* @param syncDbInfos
|
||||
* @throws SchedulerException
|
||||
*/
|
||||
public static void initSyncTable(List<SyncDbInfo> syncDbInfos) throws SchedulerException {
|
||||
String sql = "SELECT * FROM table_sync_info group by db_id,table_name order by id";
|
||||
Map<Long,SyncDbInfo> syncDbIdMap = new HashMap<Long,SyncDbInfo>();
|
||||
//key -> dbId-tableName
|
||||
Map<String,List<SyncDbInfo>> toFromMap = new LinkedHashMap<String,List<SyncDbInfo>>();
|
||||
for(SyncDbInfo dbInfo :syncDbInfos) {
|
||||
List<TableSyncInfo> tableList = new TableSyncInfo().use(dbInfo.getDbKey()).find(sql);
|
||||
syncDbIdMap.put(dbInfo.getId(), dbInfo);
|
||||
if(tableList != null && tableList.size() > 0) {
|
||||
for(TableSyncInfo tableInfo : tableList) {
|
||||
Long dbId = tableInfo.getDbId();
|
||||
String tableName = tableInfo.getTableName();
|
||||
String toKey = dbId + "#@#" + tableName;
|
||||
List<SyncDbInfo> list = toFromMap.get(toKey);
|
||||
if(list == null) {
|
||||
list = new ArrayList<SyncDbInfo>();
|
||||
toFromMap.put(toKey, list);
|
||||
}
|
||||
list.add(dbInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
Integer m2sInterval = PropKit.use("config.properties").getInt("syncMaterToSlaveTime",15);
|
||||
Integer s2mInterval = PropKit.use("config.properties").getInt("syncSlaveToMaterTime",30);
|
||||
for(Entry<String,List<SyncDbInfo>> entry : toFromMap.entrySet()) {
|
||||
String toKey = entry.getKey();
|
||||
String[] split = toKey.split("#@#");
|
||||
Long dbId = Long.valueOf(split[0]);
|
||||
String tableName = split[1];
|
||||
SyncDbInfo toDb = syncDbIdMap.get(dbId);
|
||||
List<SyncDbInfo> fromDb = entry.getValue();
|
||||
String jobGroupName = toDb.getDbKey();
|
||||
Map<String,Object> dataMap= new HashMap<String,Object>();
|
||||
dataMap.put("toDb", toDb);//同步的目标数据库
|
||||
dataMap.put("fromDb", fromDb);//从哪几个库同步
|
||||
dataMap.put("tableName", tableName);//同步的表名
|
||||
boolean master = toDb.isMaster();//是否是主库
|
||||
Integer jobInterval = master ? s2mInterval : m2sInterval;
|
||||
QuartzUtils.addJob(SCHEDULER, SyncDataJob.class, jobInterval, tableName, jobGroupName, tableName, jobGroupName, dataMap);
|
||||
logger.info(String.format("同步定时任务添加成功，toDb: %s ,toDbIsMaster: %s ,fromDbSize: %s ,interval: %ss,tableName: %s ", toDb.getDbKey(),toDb.isMaster(), fromDb.size(),jobInterval, tableName));
|
||||
}
|
||||
SCHEDULER.start();
|
||||
}
|
||||
|
||||
|
||||
|
||||
//初始化多数据源数据库连接
|
||||
public static void createConn(List<SyncDbInfo> syncDbInfos){
|
||||
logger.info("开始创建各分库数据库的连接池");
|
||||
PropKit.use("db.properties");
|
||||
for (SyncDbInfo syncDbInfo : syncDbInfos) {
|
||||
String url="jdbc:mysql://"+syncDbInfo.get("ip")+":"+syncDbInfo.get("port")+"/"+syncDbInfo.get("database_name")+"?useUnicode=true&characterEncoding=utf-8&useOldAliasMetadataBehavior=true&rewriteBatchedStatements=true";
|
||||
//初始化各数据源插件
|
||||
DruidPlugin druid=new DruidPlugin(url,(String)syncDbInfo.get("user"),(String)syncDbInfo.get("password"));
|
||||
druid.setInitialSize(Integer.valueOf(PropKit.get("dbInitialSize")));
|
||||
druid.setMaxActive(Integer.valueOf(PropKit.get("dbMaxActive")));
|
||||
druid.setMinIdle(Integer.valueOf(PropKit.get("dbMinIdle")));
|
||||
druid.setMaxWait(Integer.valueOf(PropKit.get("dbMaxWait")));
|
||||
druid.setRemoveAbandoned(Boolean.valueOf(PropKit.get("dbRemoveAbandoned")));
|
||||
druid.setRemoveAbandonedTimeoutMillis(Integer.valueOf(PropKit.get("dbRemoveAbandonedTimeout")));
|
||||
druid.setConnectionProperties(SyncData.DRUID_CONFIG_FILE_PATH);//druid 配置文件路径
|
||||
ActiveRecordPlugin arp=new ActiveRecordPlugin(syncDbInfo.getDbKey(),druid);
|
||||
arp.setShowSql(Boolean.valueOf(PropKit.get("dbShowSql")));
|
||||
druid.start();
|
||||
arp.start();
|
||||
logger.info(String.format("数据源创建成功，ip: %s,port: %s,database： %s ", syncDbInfo.getIp(),syncDbInfo.getPort(),syncDbInfo.getDatabaseName()));
|
||||
}
|
||||
logger.info("创建各分库数据库的连接池完成");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -7,5 +7,25 @@ import com.nms.model.basemodel.BaseSyncDbInfo;
|
||||
*/
|
||||
@SuppressWarnings("serial")
|
||||
public class SyncDbInfo extends BaseSyncDbInfo<SyncDbInfo> {
|
||||
|
||||
public static final SyncDbInfo dao = new SyncDbInfo().dao();
|
||||
private TableSyncInfo tableInfo;
|
||||
public String getDbKey() {
|
||||
return this.getIp()+":"+this.getPort()+"-"+this.getDatabaseName();
|
||||
}
|
||||
/**
|
||||
* 是否为主库
|
||||
* @return
|
||||
*/
|
||||
public boolean isMaster() {
|
||||
return this.getMaster() == 1;
|
||||
}
|
||||
public TableSyncInfo getTableInfo() {
|
||||
return tableInfo;
|
||||
}
|
||||
public void setTableInfo(TableSyncInfo tableInfo) {
|
||||
this.tableInfo = tableInfo;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -8,4 +8,22 @@ import com.nms.model.basemodel.BaseTableSyncInfo;
|
||||
@SuppressWarnings("serial")
|
||||
public class TableSyncInfo extends BaseTableSyncInfo<TableSyncInfo> {
|
||||
public static final TableSyncInfo dao = new TableSyncInfo().dao();
|
||||
|
||||
|
||||
public String getEventStr() {
|
||||
String result = null;
|
||||
Integer event = this.getEvent();
|
||||
switch (event) {
|
||||
case 1:
|
||||
result = "insert";break;
|
||||
case 2:
|
||||
result = "update"; break;
|
||||
case 3:
|
||||
result = "delete"; break;
|
||||
default:
|
||||
result = "unknown"; break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -57,4 +57,12 @@ public abstract class BaseSyncDbInfo<M extends BaseSyncDbInfo<M>> extends Model<
|
||||
return getStr("password");
|
||||
}
|
||||
|
||||
public void setMaster(java.lang.Integer master) {
|
||||
set("master", master);
|
||||
}
|
||||
|
||||
public java.lang.Integer getMaster() {
|
||||
return getInt("master");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -17,12 +17,12 @@ public abstract class BaseTableEventLog<M extends BaseTableEventLog<M>> extends
|
||||
return getLong("id");
|
||||
}
|
||||
|
||||
public void setTable(java.lang.String table) {
|
||||
set("table", table);
|
||||
public void setTableName(java.lang.String tableName) {
|
||||
set("table_name", tableName);
|
||||
}
|
||||
|
||||
public java.lang.String getTable() {
|
||||
return getStr("table");
|
||||
public java.lang.String getTableName() {
|
||||
return getStr("table_name");
|
||||
}
|
||||
|
||||
public void setEvent(java.lang.Integer event) {
|
||||
@@ -49,4 +49,12 @@ public abstract class BaseTableEventLog<M extends BaseTableEventLog<M>> extends
|
||||
return getStr("ids");
|
||||
}
|
||||
|
||||
public void setTime(java.util.Date time) {
|
||||
set("time", time);
|
||||
}
|
||||
|
||||
public java.util.Date getTime() {
|
||||
return get("time");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -66,11 +66,11 @@ public abstract class BaseTableSyncInfo<M extends BaseTableSyncInfo<M>> extends
|
||||
}
|
||||
|
||||
public void setLastId(java.lang.Long lastId) {
|
||||
set("last_Id", lastId);
|
||||
set("last_id", lastId);
|
||||
}
|
||||
|
||||
public java.lang.Long getLastId() {
|
||||
return getLong("last_Id");
|
||||
return getLong("last_id");
|
||||
}
|
||||
|
||||
public void setLastDate(java.util.Date lastDate) {
|
||||
@@ -113,4 +113,21 @@ public abstract class BaseTableSyncInfo<M extends BaseTableSyncInfo<M>> extends
|
||||
return getStr("interceptor");
|
||||
}
|
||||
|
||||
public void setUpdateIds(java.lang.String updateIds) {
|
||||
set("update_ids", updateIds);
|
||||
}
|
||||
|
||||
public java.lang.String getUpdateIds() {
|
||||
return getStr("update_ids");
|
||||
}
|
||||
|
||||
public void setExcludeNames(java.lang.String excludeNames) {
|
||||
set("exclude_names", excludeNames);
|
||||
}
|
||||
|
||||
public java.lang.String getExcludeNames() {
|
||||
return getStr("exclude_names");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
297
nms_sync/src/com/nms/util/QuartzUtils.java
Normal file
297
nms_sync/src/com/nms/util/QuartzUtils.java
Normal file
@@ -0,0 +1,297 @@
|
||||
package com.nms.util;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.quartz.CronScheduleBuilder;
|
||||
import org.quartz.CronTrigger;
|
||||
import org.quartz.JobBuilder;
|
||||
import org.quartz.JobDataMap;
|
||||
import org.quartz.JobDetail;
|
||||
import org.quartz.JobKey;
|
||||
import org.quartz.Scheduler;
|
||||
import org.quartz.SchedulerException;
|
||||
import org.quartz.SimpleScheduleBuilder;
|
||||
import org.quartz.SimpleTrigger;
|
||||
import org.quartz.Trigger;
|
||||
import org.quartz.TriggerBuilder;
|
||||
import org.quartz.TriggerKey;
|
||||
import org.quartz.impl.StdSchedulerFactory;
|
||||
/**
|
||||
* 定时任务调度工具类
|
||||
* @author ThinkPad
|
||||
*
|
||||
*/
|
||||
@SuppressWarnings("all")
|
||||
public class QuartzUtils {
|
||||
private static Logger logger = Logger.getLogger(QuartzUtils.class);
|
||||
private static String JOB_NAME = "SYNC_JOB";
|
||||
private static String JOB_GROUP_NAME = "SYNC_JOBGROUP";
|
||||
private static String TRIGGER_NAME = "SYNC_TRIGGER";
|
||||
private static String TRIGGER_GROUP_NAME = "SYNC_TRIGGERGROUP";
|
||||
|
||||
/**
|
||||
* 创建调度器
|
||||
* @param path
|
||||
* @return
|
||||
* @throws SchedulerException
|
||||
*/
|
||||
public static Scheduler createScheduler(String path) throws SchedulerException {
|
||||
StdSchedulerFactory sf = new StdSchedulerFactory(path);
|
||||
Scheduler scheduler = sf.getScheduler();
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
|
||||
public static Scheduler createScheduler(Properties config) throws SchedulerException {
|
||||
StdSchedulerFactory sf = new StdSchedulerFactory(config);
|
||||
Scheduler scheduler = sf.getScheduler();
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 添加一个定时任务，使用默认的任务组名，触发器名，触发器组名
|
||||
* @param sched:调度器
|
||||
* @param jobClass:任务
|
||||
* @param time:时间设置，CronExpression表达式
|
||||
*/
|
||||
public static void addJob(Scheduler sched, Class jobClass, String time) {
|
||||
addJob(sched, jobClass,time,JOB_NAME,JOB_GROUP_NAME,TRIGGER_NAME,TRIGGER_GROUP_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 添加一个定时任务
|
||||
* @param sched:调度器
|
||||
* @param jobClass:任务
|
||||
* @param time:时间设置，CronExpression表达式
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
* @param triggerName:触发器名
|
||||
* @param triggerGroupName:触发器组名
|
||||
*/
|
||||
public static void addJob(Scheduler sched, Class jobClass, String time,
|
||||
String jobName, String jobGroupName, String triggerName, String triggerGroupName) {
|
||||
|
||||
JobDetail job = JobBuilder.newJob(jobClass).withIdentity(jobName, jobGroupName).build();
|
||||
CronTrigger trigger = TriggerBuilder.newTrigger().withIdentity(triggerName, triggerGroupName)
|
||||
.withSchedule(CronScheduleBuilder.cronSchedule(time)).build();
|
||||
try {
|
||||
// 返回为 null 添加失败
|
||||
Date ft = sched.scheduleJob(job, trigger);
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @Description: 添加一个定时任务，使用默认的任务组名，触发器名，触发器组名
|
||||
* @param sched:调度器
|
||||
* @param jobClass:任务
|
||||
* @param seconds:时间设置，间隔 n s 执行一次
|
||||
* @throws SchedulerException
|
||||
*/
|
||||
public static void addJob(Scheduler sched, Class jobClass, int seconds,Map<String,Object> dataMap) throws SchedulerException {
|
||||
addJob(sched, jobClass,seconds,JOB_NAME,JOB_GROUP_NAME,TRIGGER_NAME,TRIGGER_GROUP_NAME,dataMap);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 添加一个定时任务
|
||||
* @param sched:调度器
|
||||
* @param jobClass:任务
|
||||
* @param seconds:时间设置，间隔 n s 执行一次
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
* @param triggerName:触发器名
|
||||
* @param triggerGroupName:触发器组名
|
||||
* @throws SchedulerException
|
||||
*/
|
||||
public static void addJob(Scheduler sched, Class jobClass, int seconds,
|
||||
String jobName, String jobGroupName, String triggerName, String triggerGroupName,Map<String,Object> dataMap) throws SchedulerException {
|
||||
JobDetail job = JobBuilder.newJob(jobClass).withIdentity(jobName, jobGroupName).build();
|
||||
if(dataMap != null) {
|
||||
JobDataMap jobDataMap = job.getJobDataMap();
|
||||
jobDataMap.putAll(dataMap);
|
||||
}
|
||||
SimpleTrigger trigger = TriggerBuilder.newTrigger().withIdentity(triggerName, triggerGroupName).withSchedule(SimpleScheduleBuilder.repeatSecondlyForever(seconds)).build();
|
||||
JobKey jobKey = JobKey.jobKey(jobName, jobGroupName);
|
||||
boolean checkExists = sched.checkExists(jobKey);
|
||||
if(checkExists) {
|
||||
sched.deleteJob(jobKey);
|
||||
logger.debug("任务已经存在，删除成功，" + jobKey.toString());
|
||||
}
|
||||
// 返回为 null 添加失败
|
||||
Date ft = sched.scheduleJob(job, trigger);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @Description: 定义一个任务之后进行触发设定(使用默认的任务组名，触发器名，触发器组名)
|
||||
* @param sched:调度器
|
||||
* @param time
|
||||
*/
|
||||
public static void addJObLaterUse(Scheduler sched, Class jobClass, String time) {
|
||||
|
||||
addJObLaterUse(sched,jobClass,time,JOB_NAME,JOB_GROUP_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 定义一个任务之后进行触发设定
|
||||
* @param sched:调度器
|
||||
* @param time
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
*/
|
||||
|
||||
public static void addJObLaterUse(Scheduler sched, Class jobClass, String time,
|
||||
String jobName,String jobGroupName) {
|
||||
|
||||
JobDetail job = JobBuilder.newJob(jobClass).withIdentity(jobName, jobGroupName).storeDurably().build();
|
||||
|
||||
try {
|
||||
sched.addJob(job, false);
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 对已存储的任务进行scheduling(使用默认的任务组名，触发器名，触发器组名)
|
||||
* @param sched:调度器
|
||||
* @param time
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
*/
|
||||
|
||||
public static void schedulingStoredJOb(Scheduler sched, Class jobClass, String time) {
|
||||
schedulingStoredJOb(sched,jobClass,time,JOB_NAME,JOB_GROUP_NAME,TRIGGER_NAME,TRIGGER_GROUP_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 对已存储的任务进行scheduling
|
||||
* @param sched:调度器
|
||||
* @param time
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
*/
|
||||
|
||||
public static void schedulingStoredJOb(Scheduler sched, Class jobClass, String time,
|
||||
String jobName,String jobGroupName,String triggerName, String triggerGroupName) {
|
||||
Trigger trigger = TriggerBuilder.newTrigger().withIdentity(triggerName, triggerGroupName).startNow()
|
||||
.forJob(JobKey.jobKey(jobName,jobGroupName))
|
||||
.build();
|
||||
try {
|
||||
sched.scheduleJob(trigger);
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 修改一个任务的触发时间(使用默认的任务组名，触发器名，触发器组名)
|
||||
* @param sched:调度器
|
||||
* @param time
|
||||
*/
|
||||
|
||||
public static void modifyJobTime(Scheduler sched, String time) {
|
||||
modifyJobTime(sched, TRIGGER_NAME, TRIGGER_GROUP_NAME, time);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 修改一个任务的触发时间
|
||||
* @param sched:调度器
|
||||
* @param triggerName
|
||||
* @param triggerGroupName
|
||||
* @param time
|
||||
*/
|
||||
public static void modifyJobTime(Scheduler sched, String triggerName, String triggerGroupName, String time) {
|
||||
Trigger trigger = TriggerBuilder.newTrigger().withIdentity(triggerName, triggerGroupName).withSchedule(CronScheduleBuilder.cronSchedule(time)).startNow().build();
|
||||
try {
|
||||
sched.rescheduleJob(TriggerKey.triggerKey(triggerName, triggerGroupName), trigger);
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 修改一个任务(使用默认的任务组名，任务名)
|
||||
* @param sched:调度器
|
||||
*/
|
||||
|
||||
public static void modifyJob(Scheduler sched, Class jobClass) {
|
||||
modifyJob(sched,jobClass,JOB_NAME,JOB_GROUP_NAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 修改一个任务
|
||||
* @param sched:调度器
|
||||
* @param jobName:任务名
|
||||
* @param jobGroupName:任务组名
|
||||
*/
|
||||
public static void modifyJob(Scheduler sched, Class jobClass, String jobName,String jobGroupName) {
|
||||
JobDetail job1 = JobBuilder.newJob(jobClass).withIdentity(jobName,jobGroupName).build();
|
||||
try {
|
||||
sched.addJob(job1, true);
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 删除一个任务的的trigger
|
||||
* @param sched:调度器
|
||||
* @param triggerName
|
||||
* @param triggerGroupName
|
||||
*/
|
||||
public static void unschedulingJob(Scheduler sched,String triggerName, String triggerGroupName) {
|
||||
try {
|
||||
sched.unscheduleJob(TriggerKey.triggerKey(triggerName,triggerGroupName));
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description: 移除一个任务，以及任务的所有trigger
|
||||
* @param sched:调度器
|
||||
* @param jobName
|
||||
*/
|
||||
public static void removeJob(Scheduler sched, String jobName,String jobGroupName) {
|
||||
try {
|
||||
sched.deleteJob(JobKey.jobKey(jobName,jobGroupName));
|
||||
} catch (SchedulerException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description:启动所有定时任务
|
||||
* @param sched:调度器
|
||||
*/
|
||||
public static void startJobs(Scheduler sched) {
|
||||
try {
|
||||
sched.start();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description:关闭所有定时任务
|
||||
* @param sched:调度器
|
||||
*/
|
||||
public static void shutdownJobs(Scheduler sched) {
|
||||
try {
|
||||
if (!sched.isShutdown()) {
|
||||
//未传参或false：不等待执行完成便结束；true：等待任务执行完才结束
|
||||
sched.shutdown();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
165
nms_sync/src/com/nms/util/SyncCommon.java
Normal file
165
nms_sync/src/com/nms/util/SyncCommon.java
Normal file
@@ -0,0 +1,165 @@
|
||||
package com.nms.util;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.jfinal.kit.Prop;
|
||||
import com.jfinal.kit.PropKit;
|
||||
import com.jfinal.kit.StrKit;
|
||||
import com.nms.model.TableSyncInfo;
|
||||
|
||||
/**
|
||||
* 公共方法，变量
|
||||
* @author fang
|
||||
*
|
||||
*/
|
||||
public class SyncCommon {
|
||||
private static final Logger logger = Logger.getLogger(SyncCommon.class);
|
||||
|
||||
public static final String TABLE_NAME_ALIAS;//表别名
|
||||
|
||||
static {
|
||||
Prop config = PropKit.use("config.properties");
|
||||
TABLE_NAME_ALIAS = config.get("table.alias.name", "t_alias");
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* insert mode = 2 时的select 语句，不关联 table_event_log
|
||||
* @param tableName
|
||||
* @param tableInfo
|
||||
* @param idName
|
||||
* @param columns
|
||||
* @param where
|
||||
* @return
|
||||
*/
|
||||
public static StringBuilder insertModeTwoSql( TableSyncInfo tableInfo) {
|
||||
String tableName = tableInfo.getTableName();
|
||||
String idName = tableInfo.getIdName();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
StringBuilder sbSql = new StringBuilder("select ");
|
||||
sbSql.append(getColums(columns, null)).append(" from ").append(tableName).append(" ").append(SyncCommon.TABLE_NAME_ALIAS).append(" where ").append(idName).append(" > ? ");
|
||||
if(StrKit.notBlank(where)) {
|
||||
sbSql.append(" and ( ").append(where).append(" )");
|
||||
}
|
||||
sbSql.append(" order by ").append(tableInfo.getIdName()).append(" asc limit ? ");
|
||||
return sbSql;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关联 table_event_log 表 获取 表变更记录
|
||||
* @param tableInfo
|
||||
* @return
|
||||
*/
|
||||
public static StringBuilder getSelectSqlLeftJoinEventLog(TableSyncInfo tableInfo) {
|
||||
String tableName = tableInfo.getTableName();
|
||||
String idName = tableInfo.getIdName();
|
||||
String columns = tableInfo.getColumns();
|
||||
String where = tableInfo.getWhere();
|
||||
Integer event = tableInfo.getEvent();
|
||||
Integer mode = tableInfo.getMode();
|
||||
StringBuilder sbSql = new StringBuilder();
|
||||
String colums = getColums(columns, TABLE_NAME_ALIAS);
|
||||
if(event == 2 && mode == 1 && ! columns.contains("*")) {//查询更新记录时 group by 找到符合条件最后一条记录
|
||||
sbSql.append("select ").append(colums).append(",max(log.id) as tableeventlogid").append(" from ").append(tableName).append(" ").append(TABLE_NAME_ALIAS)
|
||||
.append(" left join table_event_log log on log.target_id = ").append(TABLE_NAME_ALIAS).append(".").append(idName)
|
||||
.append(" where ").append(" log.id ").append(" > ? ").append(" and log.table_name= '").append(tableInfo.getTableName()).append("' and log.event = ").append(event);
|
||||
if(StrKit.notBlank(where)) {
|
||||
sbSql.append(" and ( ").append(where).append(" )");
|
||||
}
|
||||
sbSql.append(" group by ").append(columns);
|
||||
sbSql.append(" order by ").append(" log.id ").append(" asc limit ? ");
|
||||
}else {
|
||||
sbSql.append("select ").append(getColums(columns, TABLE_NAME_ALIAS)).append(",log.id as tableeventlogid").append(" from ").append(tableName).append(" ").append(TABLE_NAME_ALIAS)
|
||||
.append(" left join table_event_log log on log.target_id = ").append(TABLE_NAME_ALIAS).append(".").append(idName)
|
||||
.append(" where ").append(" log.id ").append(" > ? ").append(" and log.table_name= '").append(tableInfo.getTableName()).append("' and log.event = ").append(event);
|
||||
if(StrKit.notBlank(where)) {
|
||||
sbSql.append(" and ( ").append(where).append(" )");
|
||||
}
|
||||
sbSql.append(" order by ").append(" log.id ").append(" asc limit ? ");
|
||||
}
|
||||
|
||||
logger.debug(String.format("sql -> %s", sbSql.toString()));
|
||||
return sbSql;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* event = 3 delete 同步查询记录语句，关联 table_event_log
|
||||
* @param tableName
|
||||
* @param tableInfo
|
||||
* @param idName
|
||||
* @param columns
|
||||
* @param where
|
||||
* @return
|
||||
*/
|
||||
public static StringBuilder delSelectLeftJoinEventLog(TableSyncInfo tableInfo) {
|
||||
String idName = tableInfo.getIdName();
|
||||
StringBuilder sbSql = new StringBuilder("select id as tableeventlogid,target_id as ");
|
||||
sbSql.append(idName).append(" from table_event_log log")
|
||||
.append(" where ").append(" log.id ").append(" > ? ").append(" and log.table_name= '").append(tableInfo.getTableName()).append("' and log.event = ").append(tableInfo.getEvent());
|
||||
sbSql.append(" order by ").append(" log.id ").append(" asc limit ? ");
|
||||
logger.debug(String.format("sql -> %s", sbSql.toString()));
|
||||
return sbSql;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取查询字段
|
||||
* @param columns
|
||||
* @param alias 表别名
|
||||
* @return
|
||||
*/
|
||||
public static String getColums(String columns,String alias) {
|
||||
if(StrKit.notBlank(columns)) {//
|
||||
StringBuffer sb = new StringBuffer();
|
||||
String[] split = columns.split(",");
|
||||
for(String s : split) {
|
||||
if(StrKit.notBlank(s)) {
|
||||
sb.append(",");
|
||||
if(StrKit.notBlank(alias)) {
|
||||
sb.append(alias);
|
||||
sb.append(".");
|
||||
}
|
||||
sb.append(s);
|
||||
}
|
||||
}
|
||||
if(sb.length()>0) {
|
||||
sb.deleteCharAt(0);
|
||||
sb.insert(0, " ");
|
||||
sb.append(" ");
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
return StrKit.isBlank(alias)?" * " : " " +alias +".* ";
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 生成 delete sql 语句
|
||||
* @param tableName
|
||||
* @param updateIds
|
||||
* @return
|
||||
*/
|
||||
public static String genDelSql(String tableName,String updateIds,String where) {
|
||||
StringBuilder sb = new StringBuilder(" delete from ");
|
||||
sb.append(tableName).append(" where ");
|
||||
String[] split = updateIds.split(",");
|
||||
StringBuilder whereSb = new StringBuilder();
|
||||
for(String s : split) {
|
||||
if(StrKit.notBlank(s)) {
|
||||
whereSb.append("and ").append(s).append(" =? ");
|
||||
}
|
||||
}
|
||||
if(StrKit.notBlank(where)) {
|
||||
whereSb.append(" and ( ").append(where).append(" )");
|
||||
}
|
||||
if(whereSb.length() > 0) {
|
||||
whereSb.delete(0, 3);
|
||||
}
|
||||
sb.append(whereSb.toString());
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user