gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG Master 共享部分扫描结果给 TFE

Browse Source 
*	TSG-2847 - TFE 新增 cmsg 共享 TSG Master 的扫描结果
*	TSG-2851 - Pangu 从 cmsg 获取 TSG Master 的扫描结果
*	TSG-2852 - DOH   从 cmsg 获取 TSG Master 的扫描结果
*	TSG-2849 - TFE DOH 功能支持 SUBID
*	合并 pangu/doh 的 maat scan 的代码为 tfe_scan
This commit is contained in:
luwenpeng
2020-08-10 18:13:44 +08:00
parent ddb1ccba12
commit fbc7b5a715
16 changed files with 351 additions and 718 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/CMakeLists.txt
View File

@@ -1,4 +1,4 @@
add_library(common src/tfe_utils.cpp src/tfe_types.cpp src/tfe_future.cpp src/tfe_http.cpp src/tfe_plugin.cpp src/tfe_rpc.cpp src/tfe_cmsg.cpp src/tfe_kafka_logger.cpp src/tfe_resource.cpp)
add_library(common src/tfe_utils.cpp src/tfe_types.cpp src/tfe_future.cpp src/tfe_http.cpp src/tfe_plugin.cpp src/tfe_rpc.cpp src/tfe_cmsg.cpp src/tfe_kafka_logger.cpp src/tfe_resource.cpp src/tfe_scan.cpp)
target_include_directories(common PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(common PUBLIC libevent-static libevent-static-openssl libevent-static-pthreads)
target_link_libraries(common PUBLIC MESA_handle_logger cjson)

16
common/include/tfe_cmsg.h
View File

@@ -61,6 +61,22 @@ enum tfe_cmsg_tlv_type
TFE_CMSG_TCP_PASSTHROUGH,
/* Share tsg scan result */
TFE_CMSG_SRC_SUB_ID, // string max size 256
TFE_CMSG_DST_SUB_ID, // string max size 256
TFE_CMSG_SRC_ASN, // string max size 64
TFE_CMSG_DST_ASN, // string max size 64
TFE_CMSG_SRC_ORGANIZATION, // string max size 256
TFE_CMSG_DST_ORGANIZATION, // string max size 256
TFE_CMSG_SRC_IP_LOCATION_COUNTRY, // string max size 256
TFE_CMSG_DST_IP_LOCATION_COUNTRY, // string max size 256
TFE_CMSG_SRC_IP_LOCATION_PROVINE, // string max size 256
TFE_CMSG_DST_IP_LOCATION_PROVINE, // string max size 256
TFE_CMSG_SRC_IP_LOCATION_CITY, // string max size 256
TFE_CMSG_DST_IP_LOCATION_CITY, // string max size 256
/* Add new cmsg here */
/* Add new cmsg here */
/* Add new cmsg here */
/* MAX cmsg num */

22
common/include/tfe_resource.h
View File

@@ -3,40 +3,20 @@
enum RESOURCE_TYPE
{
STATIC_MAAT,
DYNAMINC_MAAT,
KAFKA_LOGGER,
DEVICE_ID,
};
enum TABLE_TYPE
{
TABLE_IP_ASN_USER_DEFINED,
TABLE_IP_ASN_BUILT_IN,
TABLE_IP_LOCATION_USER_DEFINED,
TABLE_IP_LOCATION_BUILT_IN,
TABLE_SECURITY_SOURCE_ASN,
TABLE_SECURITY_DESTINATION_ASN,
TABLE_SECURITY_SOURCE_LOCATION,
TABLE_SECURITY_DESTINATION_LOCATION,
TABLE_OBJ_SUBSCRIBER_ID,
TABLE_TYPE_MAX
};
struct ip_data_table
{
int profile_id;
int ref_cnt;
char *asn;
char *organization;
char *country_full;
char *province_full;
char *city_full;
pthread_mutex_t lock;
};
int tfe_bussiness_resouce_init();
void *tfe_bussiness_resouce_get(enum RESOURCE_TYPE type);
int tfe_bussiness_tableid_get(enum TABLE_TYPE type);
void ip_table_free(struct ip_data_table *ip_asn);

11
common/include/tfe_scan.h Normal file
View File

@@ -0,0 +1,11 @@
#pragma once
#include <tfe_stream.h>
#define MAX_SCAN_RESULT 16
int tfe_scan_subscribe_id(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr);
int tfe_scan_ip_asn(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr, char **asn_server, char **asn_client);
int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr, char **location_server, char **location_client);

1
common/include/tfe_types.h
View File

@@ -159,4 +159,3 @@ struct tfe_stream_addr * tfe_stream_addr_reverse(struct tfe_stream_addr * orin);
char* tfe_string_addr_create_by_fd(int fd, enum tfe_conn_dir dir);
char * tfe_stream_addr_to_str(const struct tfe_stream_addr * addr);
int tfe_stream_addr_str_split(char* addr_str, const char** sip, const char** sport, const char** dip, const char** dport);
int tfe_stream_addr_to_address(const struct tfe_stream_addr *addr, struct ip_address *dest_ip, struct ip_address *source_ip);

1
common/include/tfe_utils.h
View File

@@ -176,4 +176,3 @@ const char * tfe_version();
int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, unsigned value);
int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, void * val, size_t len);
int tfe_decode_base64url(u_char *dst, u_char *src);
char *tfe_unescape(char *s);

151
common/src/tfe_resource.cpp
View File

@@ -1,10 +1,9 @@
#include <tfe_utils.h>
#include <tfe_resource.h>
#include <tfe_proxy.h>
#include <tfe_kafka_logger.h>
#include <cjson/cJSON.h>
#include <MESA/Maat_rule.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/Maat_rule.h>
#include <cjson/cJSON.h>
#include <tfe_kafka_logger.h>
#include <tfe_proxy.h>
#include <tfe_resource.h>
#define MAAT_INPUT_JSON 0
#define MAAT_INPUT_REDIS 1
@@ -13,13 +12,12 @@
struct maat_table_info
{
int id;
char *name;
const char *name;
Maat_plugin_EX_new_func_t *new_func;
Maat_plugin_EX_dup_func_t *dup_func;
Maat_plugin_EX_free_func_t *free_func;
};
static Maat_feather_t static_maat = NULL;
static Maat_feather_t dynamic_maat = NULL;
static tfe_kafka_logger_t *kafka_logger = NULL;
static char *device_id = NULL;
@@ -32,7 +30,7 @@ static Maat_feather_t create_maat_feather(const char *instance_name, const char
char redis_server[TFE_STRING_MAX] = {0};
char redis_port_range[TFE_STRING_MAX] = {0};
char accept_tags[TFE_STRING_MAX] = {0};
char accept_path[TFE_PATH_MAX]={0};
char accept_path[TFE_PATH_MAX] = {0};
int redis_port_begin = 0, redis_port_end = 0;
int redis_port_select = 0;
int redis_db_idx = 0;
@@ -233,129 +231,7 @@ finish:
return (char *)device_def_id;
}
static void ip_asn_table_new_cb(int table_id, const char *key, const char *table_line, MAAT_PLUGIN_EX_DATA *ad, long argl, void *argp)
{
int addr_type;
int ret = 0, profile_id = 0, is_valid = 0;
char start_ip[40], end_ip[40], asn[40] = {0};
char organization[TFE_PATH_MAX];
ret = sscanf(table_line, "%d\t%d\t%s\t%s\t%s\t%s\t%d", &profile_id, &addr_type, start_ip, end_ip, asn, organization, &is_valid);
if (ret != 7)
{
TFE_LOG_ERROR(g_default_logger, "Policy table parse ip ASN failed, ret:%d, %s", ret, table_line);
return;
}
tfe_unescape(organization);
struct ip_data_table *ip_asn = ALLOC(struct ip_data_table, 1);
memset(ip_asn, 0, sizeof(struct ip_data_table));
ip_asn->profile_id = profile_id;
ip_asn->asn = tfe_strdup(asn);
ip_asn->organization = tfe_strdup(organization);
ip_asn->ref_cnt = 1;
pthread_mutex_init(&(ip_asn->lock), NULL);
TFE_LOG_INFO(g_default_logger, "Policy table add success %d", profile_id);
*ad = ip_asn;
}
static void ip_location_table_new_cb(int table_id, const char *key, const char *table_line, MAAT_PLUGIN_EX_DATA *ad, long argl, void *argp)
{
int ret = 0, profile_id = 0, is_valid = 0;
int geoname_id = 0, addr_type = 0;
double latitude, longitude, coords;
char language[40], start_ip[40], end_ip[40];
char continent_abbr[TFE_PATH_MAX], continent_full[TFE_PATH_MAX];
char country_abbr[TFE_PATH_MAX], province_abbr[TFE_PATH_MAX], time_zone[TFE_PATH_MAX];
char country_full[TFE_PATH_MAX], province_full[TFE_PATH_MAX], city_full[TFE_PATH_MAX];
ret = sscanf(table_line, "%d\t%d\t%d\t%s\t%s\t%lf\t%lf\t%lf\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%d", &profile_id, &geoname_id,
&addr_type, start_ip, end_ip, &latitude, &longitude, &coords, language,
continent_abbr, continent_full, country_abbr, country_full, province_abbr, province_full,
city_full, time_zone, &is_valid);
if (ret != 18)
{
TFE_LOG_ERROR(g_default_logger, "Policy table parse ip location failed, ret:%d, %s", ret, table_line);
return;
}
tfe_unescape(continent_full);
tfe_unescape(country_full);
tfe_unescape(province_full);
tfe_unescape(city_full);
struct ip_data_table *ip_asn = ALLOC(struct ip_data_table, 1);
memset(ip_asn, 0, sizeof(struct ip_data_table));
ip_asn->profile_id = profile_id;
ip_asn->country_full = tfe_strdup(country_full);
ip_asn->province_full = tfe_strdup(province_full);
ip_asn->city_full = tfe_strdup(city_full);
ip_asn->ref_cnt = 1;
pthread_mutex_init(&(ip_asn->lock), NULL);
TFE_LOG_INFO(g_default_logger, "Policy table add success %d", profile_id);
*ad = ip_asn;
}
static void ip_table_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct ip_data_table *ip_asn = (struct ip_data_table *)(*from);
pthread_mutex_lock(&(ip_asn->lock));
ip_asn->ref_cnt++;
pthread_mutex_unlock(&(ip_asn->lock));
*to = ip_asn;
}
static void ip_table_free_cb(int table_id, MAAT_PLUGIN_EX_DATA *ad, long argl, void *argp)
{
if (*ad == NULL)
{
return;
}
struct ip_data_table *ip_asn = (struct ip_data_table *)(*ad);
pthread_mutex_lock(&(ip_asn->lock));
ip_asn->ref_cnt--;
if (ip_asn->ref_cnt > 0)
{
pthread_mutex_unlock(&(ip_asn->lock));
return;
}
pthread_mutex_unlock(&(ip_asn->lock));
pthread_mutex_destroy(&(ip_asn->lock));
if (ip_asn->asn)
FREE(&ip_asn->asn);
if (ip_asn->organization)
FREE(&ip_asn->organization);
if (ip_asn->country_full)
FREE(&ip_asn->country_full);
if (ip_asn->province_full)
FREE(&ip_asn->province_full);
if (ip_asn->city_full)
FREE(&ip_asn->city_full);
FREE(&ip_asn);
*ad = NULL;
return;
}
void ip_table_free(struct ip_data_table *ip_asn)
{
ip_table_free_cb(0, (void **)&ip_asn, 0, NULL);
}
static struct maat_table_info maat_pub_tables[TABLE_TYPE_MAX] = {
// TABLE_IP_ASN_USER_DEFINED
{0, "TSG_IP_ASN_USER_DEFINED", ip_asn_table_new_cb, ip_table_dup_cb, ip_table_free_cb},
// TABLE_IP_ASN_BUILT_IN
{0, "TSG_IP_ASN_BUILT_IN", ip_asn_table_new_cb, ip_table_dup_cb, ip_table_free_cb},
// TABLE_IP_LOCATION_USER_DEFINED
{0, "TSG_IP_LOCATION_USER_DEFINED", ip_location_table_new_cb, ip_table_dup_cb, ip_table_free_cb},
// TABLE_IP_LOCATION_BUILT_IN
{0, "TSG_IP_LOCATION_BUILT_IN", ip_location_table_new_cb, ip_table_dup_cb, ip_table_free_cb},
// TABLE_SECURITY_SOURCE_ASN
{0, "TSG_SECURITY_SOURCE_ASN", NULL, NULL, NULL},
// TABLE_SECURITY_DESTINATION_ASN
@@ -363,8 +239,9 @@ static struct maat_table_info maat_pub_tables[TABLE_TYPE_MAX] = {
// TABLE_SECURITY_SOURCE_LOCATION
{0, "TSG_SECURITY_SOURCE_LOCATION", NULL, NULL, NULL},
// TABLE_SECURITY_DESTINATION_LOCATION
{0, "TSG_SECURITY_DESTINATION_LOCATION", NULL, NULL, NULL}
};
{0, "TSG_SECURITY_DESTINATION_LOCATION", NULL, NULL, NULL},
// TABLE_OBJ_SUBSCRIBER_ID
{0, "TSG_OBJ_SUBSCRIBER_ID", NULL, NULL, NULL}};
static int register_maat_table()
{
@@ -397,12 +274,6 @@ int tfe_bussiness_resouce_init()
return -1;
}
dynamic_maat = create_maat_feather("dyn", profile_path, "DYNAMIC_MAAT", thread_num, g_default_logger);
if (!dynamic_maat)
{
return -1;
}
kafka_logger = create_kafka_logger(profile_path, "kafka", g_default_logger);
if (!kafka_logger)
{
@@ -425,8 +296,6 @@ void *tfe_bussiness_resouce_get(enum RESOURCE_TYPE type)
{
case STATIC_MAAT:
return static_maat;
case DYNAMINC_MAAT:
return dynamic_maat;
case KAFKA_LOGGER:
return kafka_logger;
case DEVICE_ID:

251
common/src/tfe_scan.cpp Normal file
View File

@@ -0,0 +1,251 @@
#include <MESA/Maat_rule.h>
#include <tfe_resource.h>
#include <tfe_scan.h>
int tfe_scan_subscribe_id(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr)
{
int scan_ret = 0;
int hit_cnt_ip = 0;
uint16_t opt_out_size;
char dest_subscribe_id[TFE_STRING_MAX] = {0};
char source_subscribe_id[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_SUB_ID, (unsigned char *)source_subscribe_id, sizeof(source_subscribe_id), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src sub id from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_SUB_ID, (unsigned char *)dest_subscribe_id, sizeof(dest_subscribe_id), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst sub id from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
}
TFE_LOG_DEBUG(logger, "fetch src sub id:%s dst sub id:%s addr: %s", source_subscribe_id, dest_subscribe_id, addr);
if (strlen(source_subscribe_id))
{
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_OBJ_SUBSCRIBER_ID),
CHARSET_UTF8, source_subscribe_id, strlen(source_subscribe_id),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan src TSG_OBJ_SUBSCRIBER_ID, Hit subid: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
source_subscribe_id, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan src TSG_OBJ_SUBSCRIBER_ID, NO hit subid: %s scan ret: %d addr: %s",
source_subscribe_id, scan_ret, addr);
}
}
if (strlen(dest_subscribe_id))
{
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_OBJ_SUBSCRIBER_ID),
CHARSET_UTF8, dest_subscribe_id, strlen(dest_subscribe_id),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, Hit subid: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
dest_subscribe_id, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, NO hit subid: %s scan ret: %d addr: %s",
dest_subscribe_id, scan_ret, addr);
}
}
return hit_cnt_ip;
}
int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr, char **location_server, char **location_client)
{
int scan_ret = 0;
int hit_cnt_ip = 0;
uint16_t opt_out_size;
char buff[TFE_STRING_MAX] = {0};
char src_city[TFE_STRING_MAX] = {0};
char dst_city[TFE_STRING_MAX] = {0};
char src_provine[TFE_STRING_MAX] = {0};
char dst_provine[TFE_STRING_MAX] = {0};
char src_country[TFE_STRING_MAX] = {0};
char dst_country[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_COUNTRY, (unsigned char *)src_country, sizeof(src_country), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src country from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_COUNTRY, (unsigned char *)dst_country, sizeof(dst_country), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst country from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_PROVINE, (unsigned char *)src_provine, sizeof(src_provine), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src provine from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_PROVINE, (unsigned char *)dst_provine, sizeof(dst_provine), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst provine from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_CITY, (unsigned char *)src_city, sizeof(src_city), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src city from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_CITY, (unsigned char *)dst_city, sizeof(dst_city), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst city from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
}
TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; dst country:%s provine:%s city:%s addr: %s", src_country, src_provine, src_city, dst_country, dst_provine, dst_city, addr);
if (strlen(dst_country) || strlen(dst_city))
{
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city);
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_DESTINATION_LOCATION, Hit location: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
buff, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_DESTINATION_LOCATION, NO hit location: %s scan ret: %d addr: %s",
buff, scan_ret, addr);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", dst_city, dst_provine, dst_country);
*location_server = tfe_strdup(buff);
}
if (strlen(src_country) || strlen(src_city))
{
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city);
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_LOCATION, Hit location: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
buff, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_LOCATION, NO hit location: %s scan ret: %d addr: %s",
buff, scan_ret, addr);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", src_city, src_provine, src_country);
*location_client = tfe_strdup(buff);
}
return hit_cnt_ip;
}
int tfe_scan_ip_asn(const struct tfe_stream *stream, struct Maat_rule_t *result, scan_status_t *scan_mid,
int hit_cnt, unsigned int thread_id, void *logger, const char *addr, char **asn_server, char **asn_client)
{
int scan_ret = 0;
int hit_cnt_ip = 0;
uint16_t opt_out_size;
char buff[TFE_STRING_MAX] = {0};
char src_asn[TFE_STRING_MAX] = {0};
char dst_asn[TFE_STRING_MAX] = {0};
char src_org[TFE_STRING_MAX] = {0};
char dst_org[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_ASN, (unsigned char *)src_asn, sizeof(src_asn), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src asn from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_ASN, (unsigned char *)dst_asn, sizeof(dst_asn), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst asn from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_ORGANIZATION, (unsigned char *)src_org, sizeof(src_org), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch src org from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_ORGANIZATION, (unsigned char *)dst_org, sizeof(dst_org), &opt_out_size);
if (scan_ret != 0)
{
TFE_LOG_ERROR(logger, "fetch dst org from cmsg failed, ret: %d addr: %s", scan_ret, addr);
}
}
TFE_LOG_DEBUG(logger, "fetch src asn:%s org:%s; dst asn:%s org:%s addr: %s", src_asn, src_org, dst_asn, dst_org, addr);
if (strlen(dst_asn))
{
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_ASN),
CHARSET_UTF8, dst_asn, strlen(dst_asn),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_DESTINATION_ASN, Hit asn: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
dst_asn, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_DESTINATION_ASN, NO hit asn: %s scan ret: %d addr: %s",
dst_asn, scan_ret, addr);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s(%s)", dst_asn, dst_org);
*asn_server = tfe_strdup(buff);
}
if (strlen(src_asn))
{
scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_ASN),
CHARSET_UTF8, src_asn, strlen(src_asn),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
scan_mid, (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_ASN, Hit asn: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
src_asn, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, addr);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(logger, "Scan TSG_SECURITY_SOURCE_ASN, NO hit asn: %s scan ret: %d addr: %s",
src_asn, scan_ret, addr);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s(%s)", src_asn, src_org);
*asn_client = tfe_strdup(buff);
}
return hit_cnt_ip;
}

24
common/src/tfe_types.cpp
View File

@@ -203,27 +203,3 @@ char * tfe_string_addr_create_by_fd(int fd, enum tfe_conn_dir dir)
tfe_stream_addr_free(stream_addr);
return addr_str;
}
int tfe_stream_addr_to_address(const struct tfe_stream_addr *addr, struct ip_address *dest_ip, struct ip_address *source_ip)
{
if(addr==NULL) return -1;
if (addr->addrtype == TFE_ADDR_STREAM_TUPLE4_V4)
{
const struct tfe_stream_addr_tuple4_v4 * tuple4_v4 = addr->tuple4_v4;
source_ip->ip_type=4;
source_ip->ipv4=tuple4_v4->saddr.s_addr;
dest_ip->ip_type=4;
dest_ip->ipv4=tuple4_v4->daddr.s_addr;
}
if (addr->addrtype == TFE_ADDR_STREAM_TUPLE4_V6)
{
const struct tfe_stream_addr_tuple4_v6 * tuple4_v6 = addr->tuple4_v6;
source_ip->ip_type=6;
memcpy((char *)(source_ip->ipv6), tuple4_v6->saddr.__in6_u.__u6_addr32, IPV6_ADDR_LEN);
dest_ip->ip_type=6;
memcpy((char *)(dest_ip->ipv6),tuple4_v6->daddr.__in6_u.__u6_addr32, IPV6_ADDR_LEN);
}
return 0;
}

37
common/src/tfe_utils.cpp
View File

@@ -227,40 +227,3 @@ int tfe_decode_base64url(u_char *dst, u_char *src)
return tfe_decode_base64_internal(dst, src, basis64);
}
char *tfe_unescape(char *s)
{
int i=0,j=0;
int len=strlen(s);
for(i=0,j=0;i<len;i++)
{
if(s[i]=='\\')
{
switch(s[i+1])
{
case '&':
s[j]='&';
break;
case 'b':
s[j]=' ';//space,0x20;
break;
case '\\':
s[j]='\\';
break;
default:
s[j]=s[i];
i--; //undo the followed i++
break;
}
i++;
j++;
}
else
{
s[j]=s[i];
j++;
}
}
s[j]='\0';
return s;
}

22
conf/tfe/tfe.conf
View File

@@ -126,25 +126,3 @@ maat_redis_db_index=4
# iris mode conf iterm
full_cfg_dir=pangu_policy/full/index/
inc_cfg_dir=pangu_policy/inc/index/
[dynamic_maat]
# 0:json 1:redis 2:iris
maat_input_mode=1
stat_switch=1
perf_switch=1
table_info=resource/pangu/dynamic_maat_table_info.conf
accept_tags={"tags":[{"tag":"location","value":"Astana"}]}
stat_file=log/pangu_scan.fs2
effect_interval_s=1
# json mode conf iterm
json_cfg_file=resource/pangu/pangu_http.json
# redis mode conf iterm
maat_redis_server=10.4.34.4
maat_redis_port_range=6380-6389
maat_redis_db_index=4
# redis mode conf iterm
full_cfg_dir=pangu_policy/full/index/
inc_cfg_dir=pangu_policy/inc/index/

212
plugin/business/doh/src/doh.cpp
View File

@@ -1,6 +1,6 @@
#include "logger.h"
#include <tfe_scan.h>
#define MAX_SCAN_RESULT 16
#define DOH_CTX_MAGIC_NUM 20200601
#define REQ_METHOD_IS_GET(method) ((method == TFE_HTTP_METHOD_GET) ? 1 : 0)
@@ -199,172 +199,6 @@ static struct Maat_rule_t *doh_fetch_rule(Maat_rule_t *result, int result_num)
return p_result;
}
static int doh_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, struct ip_address *dip, int hit_cnt, unsigned int thread_id, struct doh_ctx *ctx)
{
int scan_ret = 0, hit_cnt_ip = 0;
char buff[TFE_STRING_MAX] = {0};
struct ip_data_table *ip_location_client = NULL, *ip_location_server = NULL;
int is_src_use_user_defined = 1;
int is_dst_use_user_defined = 1;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), sip, (void **)&ip_location_client, 1);
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), dip, (void **)&ip_location_server, 1);
if (ip_location_client == NULL)
{
is_src_use_user_defined = 0;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), sip, (void **)&ip_location_client, 1);
}
if (ip_location_server == NULL)
{
is_dst_use_user_defined = 0;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), dip, (void **)&ip_location_server, 1);
}
if (ip_location_server != NULL)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "scan dst ip use IP_LOCATION_%s, profile_id: %d ref_cnt: %d asn: %s organization: %s country: %s province: %s city: %s addr: %s",
(is_dst_use_user_defined ? "USER_DEFINED" : "BUILT_IN"), ip_location_server->profile_id, ip_location_server->ref_cnt, ip_location_server->asn,
ip_location_server->organization, ip_location_server->country_full, ip_location_server->province_full, ip_location_server->city_full, ctx->addr_string);
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_server->country_full, ip_location_server->city_full);
scan_ret = Maat_full_scan_string(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
&(ctx->scan_mid), (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_DST_LOCATION, Hit location: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
buff, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, ctx->addr_string);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_DST_LOCATION, NO hit location: %s scan ret: %d addr: %s",
buff, scan_ret, ctx->addr_string);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_server->city_full, ip_location_server->province_full, ip_location_server->country_full);
ctx->location_server = tfe_strdup(buff);
}
if (ip_location_client != NULL)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "scan src ip use IP_LOCATION_%s, profile_id: %d ref_cnt: %d asn: %s organization: %s country: %s province: %s city: %s addr: %s",
(is_src_use_user_defined ? "USER_DEFINED" : "BUILT_IN"), ip_location_client->profile_id, ip_location_client->ref_cnt, ip_location_client->asn,
ip_location_client->organization, ip_location_client->country_full, ip_location_client->province_full, ip_location_client->city_full, ctx->addr_string);
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_client->country_full, ip_location_client->city_full);
scan_ret = Maat_full_scan_string(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
&(ctx->scan_mid), (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_SRC_LOCATION, Hit location: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
buff, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, ctx->addr_string);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_SRC_LOCATION, NO hit location: %s scan ret: %d addr: %s",
buff, scan_ret, ctx->addr_string);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_client->city_full, ip_location_client->province_full, ip_location_client->country_full);
ctx->location_client = tfe_strdup(buff);
}
if (ip_location_server)
ip_table_free(ip_location_server);
if (ip_location_client)
ip_table_free(ip_location_client);
return hit_cnt_ip;
}
static int doh_ip_asn_scan(struct Maat_rule_t *result, struct ip_address *sip, struct ip_address *dip, int hit_cnt, unsigned int thread_id, struct doh_ctx *ctx)
{
int scan_ret = 0, hit_cnt_ip = 0;
char buff[TFE_STRING_MAX] = {0};
struct ip_data_table *ip_asn_client = NULL, *ip_asn_server = NULL;
int is_src_use_user_defined = 1;
int is_dst_use_user_defined = 1;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), sip, (void **)&ip_asn_client, 1);
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), dip, (void **)&ip_asn_server, 1);
if (ip_asn_client == NULL)
{
is_src_use_user_defined = 0;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), sip, (void **)&ip_asn_client, 1);
}
if (ip_asn_server == NULL)
{
is_dst_use_user_defined = 0;
Maat_ip_plugin_get_EX_data(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), dip, (void **)&ip_asn_server, 1);
}
if (ip_asn_server != NULL)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "scan dst ip use IP_ASN_%s, profile_id: %d ref_cnt: %d asn: %s organization: %s country: %s province: %s city: %s addr: %s",
(is_dst_use_user_defined ? "USER_DEFINED" : "BUILT_IN"), ip_asn_server->profile_id, ip_asn_server->ref_cnt, ip_asn_server->asn,
ip_asn_server->organization, ip_asn_server->country_full, ip_asn_server->province_full, ip_asn_server->city_full, ctx->addr_string);
scan_ret = Maat_full_scan_string(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_ASN),
CHARSET_UTF8, ip_asn_server->asn, strlen(ip_asn_server->asn),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
&(ctx->scan_mid), (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_DST_ASN, Hit asn: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
ip_asn_server->asn, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, ctx->addr_string);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_DST_ASN, NO hit asn: %s scan ret: %d addr: %s",
ip_asn_server->asn, scan_ret, ctx->addr_string);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_server->asn, ip_asn_server->organization);
ctx->asn_server = tfe_strdup(buff);
}
if (ip_asn_client != NULL)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "scan src ip use IP_ASN_%s, profile_id: %d ref_cnt: %d asn: %s organization: %s country: %s province: %s city: %s addr: %s",
(is_src_use_user_defined ? "USER_DEFINED" : "BUILT_IN"), ip_asn_client->profile_id, ip_asn_client->ref_cnt, ip_asn_client->asn,
ip_asn_client->organization, ip_asn_client->country_full, ip_asn_client->province_full, ip_asn_client->city_full, ctx->addr_string);
scan_ret = Maat_full_scan_string(g_doh_conf->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_ASN),
CHARSET_UTF8, ip_asn_client->asn, strlen(ip_asn_client->asn),
result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,
&(ctx->scan_mid), (int)thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_SRC_ASN, Hit asn: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
ip_asn_client->asn, scan_ret, result[hit_cnt + hit_cnt_ip].config_id, result[hit_cnt + hit_cnt_ip].service_id, result[hit_cnt + hit_cnt_ip].action, ctx->addr_string);
hit_cnt_ip += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_SRC_ASN, NO hit asn: %s scan ret: %d addr: %s",
ip_asn_client->asn, scan_ret, ctx->addr_string);
}
memset(buff, 0, sizeof(buff));
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_client->asn, ip_asn_client->organization);
ctx->asn_client = tfe_strdup(buff);
}
if (ip_asn_server)
ip_table_free(ip_asn_server);
if (ip_asn_client)
ip_table_free(ip_asn_client);
return hit_cnt_ip;
}
static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http_session *session, struct doh_ctx *ctx, char *qname, int qtype)
{
int hit_cnt = 0;
@@ -374,15 +208,19 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
struct Maat_rule_t *p_result = NULL;
struct Maat_rule_t result[MAX_SCAN_RESULT];
struct ip_address dest_ip, source_ip;
tfe_stream_addr_to_address(stream->addr, &dest_ip, &source_ip);
scan_ret = doh_ip_location_scan(result, &source_ip, &dest_ip, hit_cnt, ctx->thread_id, ctx);
scan_ret = tfe_scan_subscribe_id(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_doh_conf->local_logger, ctx->addr_string);
if (scan_ret > 0)
{
hit_cnt += scan_ret;
}
scan_ret = doh_ip_asn_scan(result, &source_ip, &dest_ip, hit_cnt, ctx->thread_id, ctx);
scan_ret = tfe_scan_ip_location(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_doh_conf->local_logger, ctx->addr_string, &(ctx->location_server), &(ctx->location_client));
if (scan_ret > 0)
{
hit_cnt += scan_ret;
}
scan_ret = tfe_scan_ip_asn(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_doh_conf->local_logger, ctx->addr_string, &(ctx->asn_server), &(ctx->asn_client));
if (scan_ret > 0)
{
hit_cnt += scan_ret;
@@ -396,14 +234,14 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
host, strlen(host), result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_HOST, Hit host: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
host, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit host: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
g_doh_conf->tables[TYPE_HOST].name, host, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
hit_cnt += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_HOST, NO hit host: %s scan ret: %d addr: %s",
host, scan_ret, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit host: %s scan ret: %d addr: %s",
g_doh_conf->tables[TYPE_HOST].name, host, scan_ret, ctx->addr_string);
}
}
@@ -413,14 +251,14 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
0, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_ADDR, Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
ctx->addr_string, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
g_doh_conf->tables[TYPE_ADDR].name, ctx->addr_string, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action);
hit_cnt += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_ADDR, NO hit addr: %s scan ret: %d",
ctx->addr_string, scan_ret);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit addr: %s scan ret: %d",
g_doh_conf->tables[TYPE_ADDR].name, ctx->addr_string, scan_ret);
}
// scan appid
@@ -428,14 +266,14 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
app_id, strlen(app_id), result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_APPID, Hit proto: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
app_id, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit proto: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
hit_cnt += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_APPID, NO hit proto: %s scan ret: %d addr: %s",
app_id, scan_ret, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit proto: %s scan ret: %d addr: %s",
g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, ctx->addr_string);
}
// scan qname
@@ -443,14 +281,14 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
qname, strlen(qname), result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id);
if (scan_ret > 0)
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_QNAME, Hit domain: %s scan ret: %d qtype: %d policy_id: %d service: %d action: %d addr: %s",
qname, scan_ret, qtype, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit domain: %s scan ret: %d qtype: %d policy_id: %d service: %d action: %d addr: %s",
g_doh_conf->tables[TYPE_QNAME].name, qname, scan_ret, qtype, result[hit_cnt].config_id, result[hit_cnt].service_id, result[hit_cnt].action, ctx->addr_string);
hit_cnt += scan_ret;
}
else
{
TFE_LOG_INFO(g_doh_conf->local_logger, "SCAN_QNAME, NO hit domain: %s scan ret: %d addr: %s",
qname, scan_ret, ctx->addr_string);
TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit domain: %s scan ret: %d addr: %s",
g_doh_conf->tables[TYPE_QNAME].name, qname, scan_ret, ctx->addr_string);
}
if (hit_cnt)

1
plugin/business/doh/src/pub.h
View File

@@ -7,7 +7,6 @@ extern "C"
#endif
#include <ctype.h>
#include <stdlib.h>
#include <event2/buffer.h>
#include <tfe_resource.h>
#include <tfe_proxy.h>

231
plugin/business/pangu-http/src/pangu_http.cpp
View File

@@ -8,6 +8,7 @@
#include <tfe_http.h>
#include <tfe_plugin.h>
#include <tfe_resource.h>
#include <tfe_scan.h>
#include <MESA/Maat_rule.h>
#include <MESA/MESA_handle_logger.h>
@@ -28,7 +29,6 @@
#include "ratelimiter.h"
#define MAX_SCAN_RESULT 16
#define MAX_EDIT_ZONE_NUM 64
enum pangu_action //Bigger action number is prior.
@@ -64,7 +64,6 @@ enum scan_table
PXY_CTRL_HTTP_REQ_BODY,
PXY_CTRL_HTTP_RES_HDR,
PXY_CTRL_HTTP_RES_BODY,
PXY_CTRL_SUBSCRIBE_ID,
PXY_CTRL_APP_ID,
__SCAN_TABLE_MAX
};
@@ -127,8 +126,6 @@ struct policy_action_param
struct pangu_rt
{
Maat_feather_t maat;
Maat_feather_t dyn_maat;
int subscriber_id_table_id;
struct pangu_logger * send_logger;
void * local_logger;
int log_level;
@@ -297,57 +294,6 @@ void trusted_CA_update_finish_cb(void* u_para)
}
}
static int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len)
{
const char* seps=" \t";
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
char* dup_line=tfe_strdup(line);
int i=0, ret=-1;
for (str = dup_line; ; str = NULL)
{
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
if(i==column_seq-1)
{
*offset=subtoken-dup_line;
*len=strlen(subtoken);
ret=0;
break;
}
i++;
}
free(dup_line);
return ret;
}
void subscribe_id_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA* to, MAAT_PLUGIN_EX_DATA* from, long argl, void* argp)
{
*to = tfe_strdup((char*)*from);
return;
}
void subscribe_id_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
int ret=0;
size_t subscribe_id_offset, len;
ret=get_column_pos(table_line, 4, &subscribe_id_offset, &len);
if(ret<0)
{
TFE_LOG_ERROR(g_pangu_rt->local_logger, "Add subscribe ID faild: %s", table_line);
return;
}
*ad=ALLOC(char, len+1);
memcpy(*ad, table_line+subscribe_id_offset, len);
TFE_LOG_INFO(g_pangu_rt->local_logger, "Add subscribe ID: %s", (char*)*ad);
return;
}
void subscribe_id_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
TFE_LOG_INFO(g_pangu_rt->local_logger, "Delete subscribe ID: %s", (char*)*ad);
free(*ad);
*ad=NULL;
}
static enum manipulate_action manipulate_action_str2idx(const char *action_str)
{
const char *clue_action_map[__MA_ACTION_MAX];
@@ -792,7 +738,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
table_name[PXY_CTRL_HTTP_REQ_BODY] = "TSG_FIELD_HTTP_REQ_CONTENT";
table_name[PXY_CTRL_HTTP_RES_HDR] = "TSG_FIELD_HTTP_RES_HDR";
table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_FIELD_HTTP_RES_CONTENT";
table_name[PXY_CTRL_SUBSCRIBE_ID] = "TSG_OBJ_SUBSCRIBER_ID";
table_name[PXY_CTRL_APP_ID] = "TSG_OBJ_APP_ID";
for (int i = 0; i < __SCAN_TABLE_MAX; i++)
{
@@ -841,22 +786,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
}
}
g_pangu_rt->dyn_maat = (Maat_feather_t)tfe_bussiness_resouce_get(DYNAMINC_MAAT);
g_pangu_rt->subscriber_id_table_id=Maat_table_register(g_pangu_rt->dyn_maat, "TSG_DYN_SUBSCRIBER_IP");
ret=Maat_plugin_EX_register(g_pangu_rt->dyn_maat,
g_pangu_rt->subscriber_id_table_id,
subscribe_id_new_cb,
subscribe_id_free_cb,
subscribe_id_dup_cb,
NULL,
0,
NULL);
if(ret!=0)
{
TFE_LOG_ERROR(NULL, "Pangu HTTP Dynamic Maat TSG_DYN_SUBSCRIBER_IP EX data register failed.");
goto error_out;
}
error_out:
return ret;
}
@@ -2405,151 +2334,6 @@ void cache_write(const struct tfe_http_session * session, enum tfe_http_event ev
}
}
int http_ip_subscribe_id_scan(struct Maat_rule_t *result, const char* sip, const char* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx)
{
int scan_ret=0, hit_cnt_ip=0;
char* source_subscribe_id=NULL, *dest_subscribe_id=NULL;
source_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, sip);
dest_subscribe_id=(char*)Maat_plugin_get_EX_data(g_pangu_rt->dyn_maat, g_pangu_rt->subscriber_id_table_id, dip);
if(source_subscribe_id!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_SUBSCRIBE_ID],
CHARSET_UTF8, source_subscribe_id, strlen(source_subscribe_id),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
}
if(dest_subscribe_id!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_SUBSCRIBE_ID],
CHARSET_UTF8, dest_subscribe_id, strlen(dest_subscribe_id),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
}
if(source_subscribe_id!=NULL)
free(source_subscribe_id);
if(dest_subscribe_id!=NULL)
free(dest_subscribe_id);
return hit_cnt_ip;
}
int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, struct ip_address *dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx )
{
int scan_ret=0, hit_cnt_ip=0;
char buff[TFE_STRING_MAX]={0};
struct ip_data_table* ip_location_client=NULL, *ip_location_server=NULL;
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), sip, (void **)&ip_location_client, 1);
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_USER_DEFINED), dip, (void **)&ip_location_server, 1);
if (ip_location_client == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), sip, (void **)&ip_location_client, 1);
}
if (ip_location_server == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_LOCATION_BUILT_IN), dip, (void **)&ip_location_server, 1);
}
if(ip_location_server!=NULL)
{
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_server->country_full, ip_location_server->city_full);
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
memset(buff,0,sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_server->city_full, ip_location_server->province_full, ip_location_server->country_full);
ctx->ip_ctx.location_server=tfe_strdup(buff);
}
if(ip_location_client!=NULL)
{
snprintf(buff, sizeof(buff), "%s.%s.", ip_location_client->country_full, ip_location_client->city_full);
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION),
CHARSET_GBK, buff, strlen(buff),
result+hit_cnt+hit_cnt_ip, NULL, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
memset(buff,0,sizeof(buff));
snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_client->city_full, ip_location_client->province_full, ip_location_client->country_full);
ctx->ip_ctx.location_client=tfe_strdup(buff);
}
if(ip_location_server)
ip_table_free(ip_location_server);
if(ip_location_client)
ip_table_free(ip_location_client);
return hit_cnt_ip;
}
int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct ip_address* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx)
{
int scan_ret=0, hit_cnt_ip=0;
char buff[TFE_STRING_MAX]={0};
struct ip_data_table* ip_asn_client=NULL, *ip_asn_server=NULL;
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), sip, (void **)&ip_asn_client, 1);
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_USER_DEFINED), dip, (void **)&ip_asn_server, 1);
if (ip_asn_client == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), sip, (void **)&ip_asn_client, 1);
}
if (ip_asn_server == NULL)
{
Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_IP_ASN_BUILT_IN), dip, (void **)&ip_asn_server, 1);
}
if(ip_asn_server!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_ASN),
CHARSET_UTF8, ip_asn_server->asn, strlen(ip_asn_server->asn),
result+hit_cnt, NULL, MAX_SCAN_RESULT-hit_cnt,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_server->asn, ip_asn_server->organization);
ctx->ip_ctx.asn_server=tfe_strdup(buff);
}
if(ip_asn_client!=NULL)
{
scan_ret = Maat_full_scan_string(g_pangu_rt->maat, tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_ASN),
CHARSET_UTF8, ip_asn_client->asn, strlen(ip_asn_client->asn),
result+hit_cnt+hit_cnt_ip, NULL, MAX_SCAN_RESULT-hit_cnt-hit_cnt_ip,
&(ctx->scan_mid), (int) thread_id);
if(scan_ret>0)
{
hit_cnt_ip+=scan_ret;
}
snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_client->asn, ip_asn_client->organization);
ctx->ip_ctx.asn_client=tfe_strdup(buff);
}
if(ip_asn_server)
ip_table_free(ip_asn_server);
if(ip_asn_client)
ip_table_free(ip_asn_client);
return hit_cnt_ip;
}
void pangu_on_http_begin(const struct tfe_stream * stream,
const struct tfe_http_session * session, unsigned int thread_id, void ** pme)
{
@@ -2562,30 +2346,23 @@ void pangu_on_http_begin(const struct tfe_stream * stream,
struct Maat_rule_t result[MAX_SCAN_RESULT];
struct ipaddr sapp_addr;
int hit_cnt = 0, scan_ret=0;
UNUSED int tmp=0;
assert(ctx == NULL);
ATOMIC_INC(&(g_pangu_rt->stat_val[STAT_SESSION]));
ctx = pangu_http_ctx_new(thread_id);
char* addr_string=tfe_stream_addr_to_str(stream->addr);
const char* sip=NULL, *dip=NULL;
tmp=tfe_stream_addr_str_split(addr_string, &sip, NULL, &dip, NULL);
assert(tmp==0);
scan_ret = http_ip_subscribe_id_scan(result, sip, dip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_subscribe_id(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string);
if(scan_ret>0)
{
hit_cnt+=scan_ret;
}
struct ip_address dest_ip, source_ip;
tfe_stream_addr_to_address(stream->addr, &dest_ip, &source_ip);
scan_ret = http_ip_location_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_ip_location(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string, &(ctx->ip_ctx.location_server), &(ctx->ip_ctx.location_client));
if(scan_ret>0)
{
hit_cnt+=scan_ret;
}
scan_ret = http_ip_asn_scan(result, &source_ip, &dest_ip, hit_cnt, thread_id, ctx);
scan_ret = tfe_scan_ip_asn(stream, result, &(ctx->scan_mid), hit_cnt, ctx->thread_id, g_pangu_rt->local_logger, addr_string, &(ctx->ip_ctx.asn_server), &(ctx->ip_ctx.asn_client));
if(scan_ret>0)
{
hit_cnt+=scan_ret;

19
resource/pangu/dynamic_maat_table_info.conf
View File

@@ -1,19 +0,0 @@
#each collumn seperate with '\t'
#id (0~65535)
#name string
#type one of ip,expr,expr_plus,digest,intval,compile or plugin
#src_charset one of GBK,BIG5,UNICODE,UTF8
#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
#do_merege [yes/no]
#cross cache [number]
#quick mode [quickon/quickoff], default [quickoff]
#For ip/intval/digest/compile/group
#id name type
#
#For plugin table
#id name type json_descr
#
#For expr/expr_plus Table
#id name type src_charset dst_charset do_merge cross_cache quick_mode
1 TSG_DYN_SUBSCRIBER_IP plugin {"key":3,"valid":5}

20
resource/pangu/table_info.conf
View File

@@ -49,15 +49,11 @@
28 TSG_SECURITY_COMPILE plugin {"key":1,"valid":8}
29 PXY_PROFILE_TRAFFIC_MIRROR plugin {"key":1,"valid":4}
30 TSG_PROFILE_DECRYPTION plugin {"key":1,"valid":4}
31 TSG_IP_ASN_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7,"estimate_size":4194304}
32 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7,"estimate_size":4194304}
33 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18,"estimate_size":4194304}
34 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18,"estimate_size":4194304}
35 TSG_OBJ_AS_NUMBER expr UTF8 UTF8/GBK yes 0
36 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_AS_NUMBER --
37 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_AS_NUMBER --
38 TSG_OBJ_GEO_LOCATION expr UTF8 UTF8/GBK yes 0
39 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_GEO_LOCATION --
40 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION --
41 TSG_FIELD_DOH_QNAME virtual TSG_OBJ_FQDN --
42 TSG_FIELD_DOH_HOST virtual TSG_OBJ_FQDN --
31 TSG_OBJ_AS_NUMBER expr UTF8 UTF8/GBK yes 0
32 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_AS_NUMBER --
33 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_AS_NUMBER --
34 TSG_OBJ_GEO_LOCATION expr UTF8 UTF8/GBK yes 0
35 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_GEO_LOCATION --
36 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION --
37 TSG_FIELD_DOH_QNAME virtual TSG_OBJ_FQDN --
38 TSG_FIELD_DOH_HOST virtual TSG_OBJ_FQDN --