gfwleak/tango-kni
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加和tfe通信接口, 添加负载均衡功能等

Browse Source
This commit is contained in:
崔一鸣
2019-06-03 20:19:04 +08:00
parent 85aee8ba55
commit 1fa7a0673f
20 changed files with 1607 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1015
entry/src/kni_entry.cpp
View File

File diff suppressed because it is too large Load Diff

151
entry/src/kni_maat.cpp
View File

@@ -2,8 +2,16 @@
#include "kni_maat.h"
extern int g_iThreadNum;
int g_maat_default_action = -1;
struct kni_maat_handle{
Maat_feather_t feather;
int tableid_intercept_ip;
int tableid_intercept_domain;
void *logger;
};
void kni_maat_destroy(struct kni_maat_handle *handle){
if(handle != NULL){
if(handle->feather != NULL){
@@ -42,76 +50,125 @@ struct kni_maat_handle* kni_maat_init(const char* profile, void *logger){
char tablename_intercept_ip[KNI_SYMBOL_MAX];
char tablename_intercept_domain[KNI_SYMBOL_MAX];
char compile_alias[KNI_SYMBOL_MAX];
MESA_load_profile_int_def(profile, section, "readconf_mode", &readconf_mode, KNI_MAAT_READCONF_IRIS);
MESA_load_profile_string_def(profile, section, "tableinfo_path", tableinfo_path, sizeof(tableinfo_path), "unknown");
MESA_load_profile_string_def(profile, section, "tablename_intercept_ip", tablename_intercept_ip, sizeof(tablename_intercept_ip), "unknown");
MESA_load_profile_string_def(profile, section, "tablename_intercept_domain", tablename_intercept_domain, sizeof(tablename_intercept_domain), "unknown");
MESA_load_profile_string_def(profile, section, "compile_alias", compile_alias, sizeof(compile_alias), "unknown");
char maatjson_path[KNI_PATH_MAX];
char redis_ip[INET_ADDRSTRLEN];
int redis_port;
int redis_index;
Maat_feather_t feather = NULL;
int tableid_intercept_ip = -1;
int tableid_intercept_domain = -1;
struct kni_maat_handle *handle = NULL;
int ret = MESA_load_profile_int_nodef(profile, section, "readconf_mode", &readconf_mode);
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: readconf_mode not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "tableinfo_path", tableinfo_path, sizeof(tableinfo_path));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: tableinfo_path not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "tablename_intercept_ip", tablename_intercept_ip, sizeof(tablename_intercept_ip));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: tablename_intercept_ip not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "tablename_intercept_domain", tablename_intercept_domain, sizeof(tablename_intercept_domain));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: tablename_intercept_domain not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "compile_alias", compile_alias, sizeof(compile_alias));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: compile_alias not set, profile is %s, section is %s", profile, section);
goto error_out;
}
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n readconf_mode: %d\n tableinfo_path: %s\n tablename_intercept_ip: %s\n tablename_intercept_domain: %s\n"
"compile_alias: %s\n", section, readconf_mode, tableinfo_path, tablename_intercept_ip, tablename_intercept_domain, compile_alias);
Maat_feather_t feather = Maat_feather(g_iThreadNum, tableinfo_path, logger);
feather = Maat_feather(g_iThreadNum, tableinfo_path, logger);
handle = ALLOC(struct kni_maat_handle, 1);
handle->feather = feather;
if(feather == NULL){
KNI_LOG_ERROR(logger, "Failed at Maat_feather, max_thread_num is %d, tableinfo_path is %s", g_iThreadNum, tableinfo_path);
return NULL;
}
if(readconf_mode == KNI_MAAT_READCONF_JSON){
char maatjson_path[KNI_PATH_MAX];
MESA_load_profile_string_def(profile, section, "maatjson_path", maatjson_path, sizeof(maatjson_path), "unknown");
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n maatjson_path: %s", section, maatjson_path);
Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, maatjson_path, strlen(maatjson_path));
switch(readconf_mode){
case KNI_MAAT_READCONF_JSON:
ret = MESA_load_profile_string_nodef(profile, section, "maatjson_path", maatjson_path, sizeof(maatjson_path));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: maatjson_path not set, profile is %s, section is %s", profile, section);
goto error_out;
}
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n maatjson_path: %s", section, maatjson_path);
Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, maatjson_path, strlen(maatjson_path));
break;
case KNI_MAAT_READCONF_IRIS:
break;
case KNI_MAAT_READCONF_REDIS:
ret = MESA_load_profile_string_nodef(profile, section, "redis_ip", redis_ip, sizeof(redis_ip));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: redis_ip not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_int_nodef(profile, section, "redis_port", &redis_port);
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: redis_port not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_int_nodef(profile, section, "redis_index", &redis_index);
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: redis_index not set, profile is %s, section is %s", profile, section);
goto error_out;
}
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n redis_ip: %s\n redis_port: %d\n redis_index: %d",
section, redis_ip, redis_port, redis_index);
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, (void*)redis_ip, strlen(redis_ip) + 1);
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_PORT, (void*)&redis_port, sizeof(redis_port));
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_INDEX, (void*)&redis_index, sizeof(redis_index));
break;
default:
break;
}
if(readconf_mode == KNI_MAAT_READCONF_IRIS){
//TODO
}
if(readconf_mode == KNI_MAAT_READCONF_REDIS){
char redis_ip[KNI_SYMBOL_MAX];
int redis_port;
int redis_index;
MESA_load_profile_string_def(profile, section, "redis_ip", redis_ip, sizeof(redis_ip), "unknown");
MESA_load_profile_int_def(profile, section, "redis_port", &redis_port, -1);
MESA_load_profile_int_def(profile, section, "redis_index", &redis_index, -1);
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n redis_ip: %s\n redis_port: %s\n redis_index: %d",
section, redis_ip, redis_port, redis_index);
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, (void*)redis_ip, strlen(redis_ip) + 1);
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_PORT, (void*)&redis_port, sizeof(redis_port));
Maat_set_feather_opt(feather, MAAT_OPT_REDIS_INDEX, (void*)&redis_index, sizeof(redis_index));
}
int ret = Maat_initiate_feather(feather);
ret = Maat_initiate_feather(feather);
if(ret < 0){
KNI_LOG_ERROR(logger, "Failed at Maat_initiate_feather");
return NULL;
goto error_out;
}
int tableid_intercept_ip = Maat_table_register(feather, tablename_intercept_ip);
int tableid_intercept_domain = Maat_table_register(feather, tablename_intercept_domain);
tableid_intercept_ip = Maat_table_register(feather, tablename_intercept_ip);
tableid_intercept_domain = Maat_table_register(feather, tablename_intercept_domain);
if(tableid_intercept_ip < 0){
KNI_LOG_ERROR(logger, "Failed at Maat_table_register, tablename is %d, ret is %d",
tablename_intercept_ip, tableid_intercept_ip);
return NULL;
goto error_out;
}
if(tableid_intercept_domain < 0){
KNI_LOG_ERROR(logger, "Failed at Maat_table_register, tablename is %d, ret is %d",
tablename_intercept_domain, tableid_intercept_domain);
return NULL;
goto error_out;
}
struct kni_maat_handle *handle = ALLOC(struct kni_maat_handle, 1);
ret = Maat_rule_get_ex_new_index(feather, compile_alias, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, 0, logger);
if(ret < 0){
KNI_LOG_ERROR(logger, "Failed at Maat_rule_get_ex_new_index, ret is %d", ret);
kni_maat_destroy(handle);
return NULL;
goto error_out;
}
handle->feather = feather;
handle->tableid_intercept_ip = tableid_intercept_ip;
handle->tableid_intercept_domain = tableid_intercept_domain;
handle->logger = logger;
return handle;
error_out:
kni_maat_destroy(handle);
return NULL;
}
static int maat_process_scan_result(struct kni_maat_handle *handle, int num, struct Maat_rule_t *result){
static int maat_process_scan_result(struct kni_maat_handle *handle, int num, struct Maat_rule_t *result, int *policy_id){
//void *logger = handle->logger;
int action = g_maat_default_action;
*policy_id = 0; //默认动作是编译表中policy_id=0的字段，所以默认policy_id=0;
for(int i = 0; i < num; i++){
action = result[i].action;
*policy_id = result[i].config_id;
if(action == KNI_ACTION_BYPASS){
return action;
}
@@ -121,7 +178,7 @@ static int maat_process_scan_result(struct kni_maat_handle *handle, int num, str
//TODO: Maat_rule_get_ex_new_index compile_ex_param_new: config_id = 0, 取action即为全局变量, 一旦配置更新就回调, tableinfo怎么写，回调表， 编译配置表
int kni_maat_scan_ip(struct kni_maat_handle *handle, struct ipaddr *addr, int thread_seq){
int kni_maat_scan_ip(struct kni_maat_handle *handle, struct ipaddr *addr, int thread_seq, int *policy_id){
//printf("default action is %d\n", g_maat_default_action);
void *logger = handle->logger;
struct Maat_rule_t result[KNI_MAAT_RULE_NUM_MAX];
@@ -132,17 +189,17 @@ int kni_maat_scan_ip(struct kni_maat_handle *handle, struct ipaddr *addr, int th
KNI_LOG_ERROR(logger, "Failed at Maat_scan_proto_addr, ret is %d", ret);
return g_maat_default_action;
}
int action = maat_process_scan_result(handle, ret, result);
int action = maat_process_scan_result(handle, ret, result, policy_id);
//for debug
char saddr[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &(addr->v4->saddr), saddr, INET_ADDRSTRLEN);
KNI_LOG_DEBUG(logger, "ip is %s, ret is %d, action is %d\n", saddr, ret, action);
char stream_addr[KNI_SYMBOL_MAX] = "";
kni_stream_addr_trans(addr, stream_addr, sizeof(stream_addr));
KNI_LOG_DEBUG(logger, "maat_scan_ip, %s, policy_id = %d, action = %s\n",
stream_addr, *policy_id, action == KNI_ACTION_BYPASS ? "bypss" : "intercept");
return action;
}
int kni_maat_scan_domain(struct kni_maat_handle* handle, char *domain, int domain_len, int thread_seq){
int kni_maat_scan_domain(struct kni_maat_handle* handle, char *domain, int domain_len, int thread_seq, int *policy_id){
void *logger = handle->logger;
struct Maat_rule_t result[KNI_MAAT_RULE_NUM_MAX];
//必须要初始化为NULL, 不懂为什么
@@ -153,14 +210,14 @@ int kni_maat_scan_domain(struct kni_maat_handle* handle, char *domain, int domai
KNI_LOG_ERROR(logger, "Failed at Maat_full_scan_string, ret is %d", ret);
return g_maat_default_action;
}
int action = maat_process_scan_result(handle, ret, result);
int action = maat_process_scan_result(handle, ret, result, policy_id);
//for debug
char domain1[100] = "";
memcpy(domain1, domain, domain_len);
domain1[domain_len] = '\0';
KNI_LOG_DEBUG(logger, "domain is %s, ret is %d, action is %d\n", domain, ret, action);
KNI_LOG_DEBUG(logger, "maat_scan_domain: %s, policy_id = %d, action = %s\n",
domain, *policy_id, action == KNI_ACTION_BYPASS ? "bypss" : "intercept");
return action;
}

149
entry/src/kni_send_logger.cpp Normal file
View File

@@ -0,0 +1,149 @@
#include "kni_utils.h"
#include "kni_send_logger.h"
#include "librdkafka/rdkafka.h"
struct kni_send_logger{
int sendlog_switch;
rd_kafka_t *kafka_handle;
rd_kafka_topic_t *kafka_topic;
void *local_logger;
};
static rd_kafka_t* kafka_init(const char *profile, void *logger){
rd_kafka_t *kafka_handle = NULL;
rd_kafka_conf_t *rdkafka_conf = NULL;
char kafka_errstr[1024];
const char *section = "kafka";
char queue_buffering_max_messages[KNI_SYMBOL_MAX] = "";
char topic_metadata_refresh_interval_ms[KNI_SYMBOL_MAX] = "";
char security_protocol[KNI_SYMBOL_MAX] = "";
int ret = MESA_load_profile_string_nodef(profile, section, "queue.buffering.max.messages",
queue_buffering_max_messages, sizeof(queue_buffering_max_messages));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: queue.buffering.max.messages not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "topic.metadata.refresh.interval.ms",
topic_metadata_refresh_interval_ms, sizeof(topic_metadata_refresh_interval_ms));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: topic.metadata.refresh.interval.ms not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "security.protocol", security_protocol, sizeof(security_protocol));
if(ret < 0){
KNI_LOG_ERROR(logger, "MESA_prof_load: security.protocol not set, profile is %s, section is %s", profile, section);
goto error_out;
}
KNI_LOG_INFO(logger, "MESA_prof_load, [%s]:\n queue.buffering.max.messages: %s\n topic.metadata.refresh.interval.ms: %s\n"
"security.protocol: %s", "kafka", queue_buffering_max_messages, topic_metadata_refresh_interval_ms, security_protocol);
rdkafka_conf = rd_kafka_conf_new();
rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", queue_buffering_max_messages, kafka_errstr, sizeof(kafka_errstr));
rd_kafka_conf_set(rdkafka_conf, "topic.metadata.refresh.interval.ms", topic_metadata_refresh_interval_ms, kafka_errstr, sizeof(kafka_errstr));
rd_kafka_conf_set(rdkafka_conf, "security.protocol", security_protocol, kafka_errstr, sizeof(kafka_errstr));
//The conf object is freed by this function and must not be used or destroyed by the application sub-sequently.
kafka_handle = rd_kafka_new(RD_KAFKA_PRODUCER, rdkafka_conf, kafka_errstr, sizeof(kafka_errstr));
rdkafka_conf = NULL;
if(kafka_handle == NULL){
goto error_out;
}
return kafka_handle;
error_out:
if(rdkafka_conf != NULL){
rd_kafka_conf_destroy(rdkafka_conf);
rdkafka_conf = NULL;
}
if(kafka_handle != NULL){
rd_kafka_destroy(kafka_handle);
kafka_handle = NULL;
}
return NULL;
}
void kni_send_logger_destroy(struct kni_send_logger *handle){
if(handle != NULL){
if(handle->kafka_topic != NULL){
rd_kafka_topic_destroy(handle->kafka_topic);
handle->kafka_topic = NULL;
}
if(handle->kafka_handle != NULL){
rd_kafka_destroy(handle->kafka_handle);
handle->kafka_handle = NULL;
}
FREE(&handle);
}
}
struct kni_send_logger* kni_send_logger_init(const char *profile, void *local_logger){
struct kni_send_logger *handle = NULL;
const char *section = "send_logger";
int sendlog_switch = -1;
char kafka_topic[KNI_SYMBOL_MAX] = "";
char kafka_brokerlist[KNI_SYMBOL_MAX] = "";
rd_kafka_t *kafka_handle = NULL;
rd_kafka_topic_t *topic = NULL;
int ret = MESA_load_profile_int_nodef(profile, section, "switch", &sendlog_switch);
if(ret < 0){
KNI_LOG_ERROR(local_logger, "MESA_prof_load: switch not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "kafka_topic", kafka_topic, sizeof(kafka_topic));
if(ret < 0){
KNI_LOG_ERROR(local_logger, "MESA_prof_load: kafka_topic not set, profile is %s, section is %s", profile, section);
goto error_out;
}
ret = MESA_load_profile_string_nodef(profile, section, "kafka_brokerlist", kafka_brokerlist, sizeof(kafka_brokerlist));
if(ret < 0){
KNI_LOG_ERROR(local_logger, "MESA_prof_load: kafka_brokerlist not set, profile is %s, section is %s", profile, section);
goto error_out;
}
KNI_LOG_INFO(local_logger, "MESA_prof_load, [%s]:\n switch: %d\n kafka_topic: %s\n, kafka_brokerlist: %s",
section, sendlog_switch, kafka_topic, kafka_brokerlist);
handle = ALLOC(struct kni_send_logger, 1);
handle->local_logger = local_logger;
//sendlog_switch = 0, 不发送日志给kafka
if(sendlog_switch == 0){
handle->sendlog_switch = 0;
return handle;
}
handle->sendlog_switch = 1;
//init kafka
kafka_handle = kafka_init(profile, local_logger);
if(kafka_handle == NULL){
KNI_LOG_ERROR(local_logger, "Failed at init kafka");
goto error_out;
}
handle->kafka_handle = kafka_handle;
//kafka_brokerlist
ret = rd_kafka_brokers_add(kafka_handle, kafka_brokerlist);
if(ret == 0){
KNI_LOG_ERROR(local_logger, "Failed at add kafka_brokers");
goto error_out;
}
//kafka topic
topic = rd_kafka_topic_new(kafka_handle, kafka_topic, NULL);
if(topic == NULL){
KNI_LOG_ERROR(local_logger, "Failed at new kafka topic");
goto error_out;
}
handle->kafka_topic = topic;
return handle;
error_out:
kni_send_logger_destroy(handle);
return NULL;
}
int kni_send_logger_sendlog(kni_send_logger *handle, char *log_msg, int log_msg_len){
void *logger = handle->local_logger;
//kafka produce
int kafka_status = rd_kafka_produce(handle->kafka_topic, RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY,
log_msg, log_msg_len, NULL, 0, NULL);
if(kafka_status < 0){
KNI_LOG_ERROR(logger, "Kafka: Failed to produce, error is %s", rd_kafka_err2name(rd_kafka_last_error()));
return -1;
}
return 0;
}