gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-22316 Manipulation支持Device相关策略的扫描与日志发送, TSG-22051 Manipulation日志发送Library相关字段, TSG-22256 Manipulation日志发送Incoming Link ID和Outgoing Link ID等字段

Browse Source
This commit is contained in:
fengweihao
2024-08-30 10:25:16 +08:00
parent 9e2b08ab53
commit 80eed59202
11 changed files with 529 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/src/packet.cpp
View File

@@ -42,8 +42,8 @@
* Static API
******************************************************************************/
static const char *ethproto_tostring(uint16_t proto);
static const char *ipproto_tostring(uint16_t proto);
static const char *ethproto_tostring(uint16_t proto)__attribute__((unused));
static const char *ipproto_tostring(uint16_t proto)__attribute__((unused));
static inline const char *ldbc_method_tostring(enum ldbc_method method);
static inline const char *layer_type_tostring(enum layer_type type);

2
common/src/tfe_ctrl_packet.cpp
View File

@@ -64,7 +64,7 @@ int tags_ids_cmsg_maps[] = {
[INDEX_FQDN_TAGS_IDS] = TFE_CMSG_FQDN_TAGS_IDS_STR,
};
char *tags_ids_cmsg_name_maps[] = {
const char *tags_ids_cmsg_name_maps[] = {
[INDEX_SRC_IP_TAGS_IDS] = "TFE_CMSG_SRC_IP_TAGS_IDS_STR",
[INDEX_DST_IP_TAGS_IDS] = "TFE_CMSG_DST_IP_TAGS_IDS_STR",
[INDEX_FQDN_TAGS_IDS] = "TFE_CMSG_FQDN_TAGS_IDS_STR",

113
common/src/tfe_resource.cpp
View File

@@ -319,6 +319,110 @@ void app_dict_table_dup_cb(int table_id, void **to, void **from, long argl, void
return;
}
int get_category_type_str2idx(const char *category)
{
size_t i = 0;
const char *category_name[] = {"unknown", "geoip_city", "geoip_country", "geoip_asn", "website_classfication", "internet_service", "security_threat", "compliance_risk"};
for (i = 0; i < sizeof(category_name) / sizeof(const char *); i++)
{
if (0 == strcasecmp(category, category_name[i]))
break;
}
return i;
}
void library_tag_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp)
{
int ret=0;
size_t offset=0, len=0;
char category[256]={0};
struct library_tag_ctx *library_tags = ALLOC(struct library_tag_ctx, 1);
ret = maat_helper_read_column(table_line, 1, &offset, &len);
if(ret >= 0)
{
char *tag_id_str=ALLOC(char, len+1);
memcpy(tag_id_str, table_line+offset, len);
library_tags->tag_id=atoi(tag_id_str);
FREE(&tag_id_str);
}
ret = maat_helper_read_column(table_line, 3, &offset, &len);
if(ret >= 0)
{
memcpy(category, table_line+offset, len);
library_tags->category=(enum category_type)get_category_type_str2idx(category);
}
ret = maat_helper_read_column(table_line, 4, &offset, &len);
if(ret >= 0)
{
library_tags->tag_key=ALLOC(char, len+1);
memcpy(library_tags->tag_key, table_line+offset, len);
}
ret = maat_helper_read_column(table_line, 5, &offset, &len);
if(ret >= 0)
{
library_tags->tag_value=ALLOC(char, len+1);
memcpy(library_tags->tag_value, table_line+offset, len);
}
library_tags->ref_cnt=1;
pthread_mutex_init(&(library_tags->lock), NULL);
*ad=library_tags;
return;
}
void library_tag_free_cb(int table_id, void **ad, long argl, void* argp)
{
if(*ad==NULL)
{
return;
}
struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*ad);
pthread_mutex_lock(&(library_tags->lock));
library_tags->ref_cnt--;
if(library_tags->ref_cnt>0)
{
pthread_mutex_unlock(&(library_tags->lock));
return;
}
pthread_mutex_unlock(&(library_tags->lock));
pthread_mutex_destroy(&(library_tags->lock));
if(library_tags->tag_key)
{
FREE(&library_tags->tag_key);
}
if(library_tags->tag_value)
{
FREE(&library_tags->tag_value);
}
FREE(&library_tags);
*ad=NULL;
return;
}
void library_tag_dup_cb(int table_id, void **to, void **from, long argl, void* argp)
{
struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*from);
pthread_mutex_lock(&(library_tags->lock));
library_tags->ref_cnt++;
pthread_mutex_unlock(&(library_tags->lock));
*to=library_tags;
return;
}
void library_tag_free(struct library_tag_ctx *library_tags)
{
library_tag_free_cb(0, (void **)&library_tags, 0, NULL);
}
static int maat_common_table_init()
{
const char * table_name[__SCAN_COMMON_TABLE_MAX];
@@ -333,6 +437,11 @@ static int maat_common_table_init()
table_name[PXY_CTRL_IP_PROTOCOL] = "ATTR_IP_PROTOCOL";
table_name[PXY_CTRL_SUBSCRIBER_ID] = "ATTR_SUBSCRIBER_ID";
table_name[PXY_CTRL_APP_ID_DICT] = "APP_ID_DICT";
table_name[PXY_CTRL_LIBRARY_TAG] = "LIBRARY_TAG";
table_name[PXY_CTRL_IMSI]="ATTR_GTP_IMSI";
table_name[PXY_CTRL_APN]="ATTR_GTP_APN";
table_name[PXY_CTRL_PHONE_NUMBER]="ATTR_GTP_PHONE_NUMBER";
table_name[PXY_CTRL_GTP_IMEI]="ATTR_GTP_IMEI";
for (int i = 0; i < __SCAN_COMMON_TABLE_MAX; i++)
{
@@ -344,7 +453,9 @@ static int maat_common_table_init()
}
}
maat_plugin_table_ex_schema_register(maat_handle, "APP_ID_DICT", app_dict_table_new_cb, app_dict_table_free_cb, app_dict_table_dup_cb, 0, NULL);
return 0;
maat_plugin_table_ex_schema_register(maat_handle, "LIBRARY_TAG", library_tag_new_cb, library_tag_free_cb, library_tag_dup_cb, 0, NULL);
return 0;
}
int tfe_env_init()

213
common/src/tfe_scan.cpp
View File

@@ -114,34 +114,79 @@ int tfe_scan_internal_exteral_by_last_group(const struct tfe_stream *stream, lon
return hit_cnt_group;
}
/**for Provisional compilation definitions**/
#define TFE_CMSG_FQDN_TAGS_LOG_STR 63 /*"Category Name:Search Engines, Cloude Provider:aws, CDN Provider:aws, category_name: ab"*/
#define TFE_CMSG_SRC_TAGS_LOG_STR 64 /*"administrative_area:Singapore, country:Singapore", CDN Provider:Light CDN*/
#define TFE_CMSG_DST_TAGS_LOG_STR 65 /*"ASN:55967, CDN Provider:Light CDN, country:Hong Kong"*/
int tfe_get_entry_tags(const struct tfe_stream * stream, enum tfe_cmsg_tlv_type tlv_type, char *opt_val, long long *tag_id_array)
{
int n_tag_ids = 0;
uint16_t opt_out_size = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
{
return 0;
}
int ret = tfe_cmsg_get_value(cmsg, tlv_type, (unsigned char *)opt_val, 128, &opt_out_size);
if(ret == 0 && opt_out_size > 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
}
return n_tag_ids;
}
int tfe_get_library_tags(const struct tfe_stream *stream, cJSON *common_obj, tfe_cmsg_tlv_type tlv_type, const char *tag_key)
{
char opt_val[128]={0};
long long tag_id_array[128]={0};
int n_tag_ids = tfe_get_entry_tags(stream, tlv_type, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return 0;
}
char value[128]={0};
cJSON *tags_array = cJSON_CreateArray();
for(int i = 0; i < n_tag_ids; i++)
{
struct library_tag_ctx *library_tag =(struct library_tag_ctx *)maat_plugin_table_get_ex_data(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_LIBRARY_TAG), (const char *)&tag_id_array[i], sizeof(long long));
if(library_tag != NULL)
{
if(library_tag->category == CATEGORY_TYPE_ASN && tlv_type == TFE_CMSG_SRC_IP_TAGS_IDS_STR)
{
cJSON_AddNumberToObject(common_obj, "client_asn", atol(library_tag->tag_value));
}
if(library_tag->category == CATEGORY_TYPE_ASN && tlv_type == TFE_CMSG_DST_IP_TAGS_IDS_STR)
{
cJSON_AddNumberToObject(common_obj, "server_asn", atol(library_tag->tag_value));
}
memset(value, 0, sizeof(value));
snprintf(value, sizeof(value), "%s:%s", library_tag->tag_key, library_tag->tag_value);
cJSON_AddItemToArray(tags_array, cJSON_CreateString(value));
}
library_tag_free(library_tag);
}
cJSON_AddItemToObject(common_obj, tag_key, tags_array);
return 0;
}
int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
size_t n_hit_result = 0;
long long tag_id_array[128]={0};
int scan_ret = 0, hit_cnt_ip = 0, n_tag_ids = 0;
char opt_val[128]={0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_SRC_IP_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
char opt_val[128]={0}; uint16_t opt_out_size = 0;
int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_SRC_IP_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
}
TFE_LOG_DEBUG(logger, "fetch src ip tags: %s", opt_val);
struct maat_hit_group hit_group;
for (int i = 0; i < n_tag_ids; i++)
{
@@ -166,18 +211,16 @@ int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct
}
}
n_tag_ids=0;
memset(opt_val, 0, sizeof(opt_val));
memset(tag_id_array, 0, sizeof(tag_id_array));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_DST_IP_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_DST_IP_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
return hit_cnt_ip;
}
TFE_LOG_DEBUG(logger, "fetch dst ip tags: %s", opt_val);
for (int i = 0; i < n_tag_ids; i++)
{
memset(&hit_group, 0, sizeof(hit_group));
@@ -204,27 +247,17 @@ int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct
int tfe_scan_fqdn_tags(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, int table_id, void *logger)
{
char opt_val[128]={0};
long long tag_id_array[128]={0};
int scan_ret = 0, hit_cnt_fqdn = 0, n_tag_ids = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_FQDN_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_fqdn;
}
char opt_val[128]={0}; uint16_t opt_out_size = 0;
int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_FQDN_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_fqdn;
}
}
TFE_LOG_DEBUG(logger, "fetch fqdn tags: %s", opt_val);
struct maat_hit_group hit_group;
for (int i = 0; i < n_tag_ids; i++)
{
@@ -241,7 +274,7 @@ int tfe_scan_fqdn_tags(const struct tfe_stream *stream, long long *result, struc
TFE_LOG_INFO(logger, "Scan Fqdn TAGS, NO hit scan ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
}
return hit_cnt_fqdn;
return hit_cnt_fqdn;
}
int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, long long app_id, int table_id)
@@ -273,6 +306,71 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt,
return hit_app_id;
}
int tfe_scan_value_by_cmsg(const struct tfe_stream *stream, enum tfe_cmsg_tlv_type tlv_type, long long *result, struct maat_state *scan_mid, int hit_cnt, int table_id, char *opt_val, void *logger)
{
uint16_t opt_out_size = 0;
int hit_cnt_string=0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
{
return hit_cnt_string;
}
int ret = tfe_cmsg_get_value(cmsg, tlv_type, (unsigned char *)opt_val, 128, &opt_out_size);
if(ret == 0 && opt_out_size > 0)
{
size_t n_hit_result=0;
int scan_ret = maat_scan_string(tfe_get_maat_handle(), table_id, opt_val, strlen(opt_val), result+hit_cnt+hit_cnt_string, MAX_SCAN_RESULT-hit_cnt-hit_cnt_string,
&n_hit_result,scan_mid);
if(scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_string+=n_hit_result;
}
scan_ret = maat_scan_not_logic(tfe_get_maat_handle(), table_id, result+hit_cnt+hit_cnt_string, MAX_SCAN_RESULT-hit_cnt-hit_cnt_string, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_string+=n_hit_result;
}
}
return hit_cnt_string;
}
int tfe_scan_device(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
char opt_val[4][128]={0};
int scan_ret = 0, htt_cnt_device = 0;
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_IMSI_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_IMSI), opt_val[0], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_IMEI_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_GTP_IMEI), opt_val[1], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_PHONE_NUM_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_PHONE_NUMBER), opt_val[2], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_APN_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_APN), opt_val[3], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
TFE_LOG_DEBUG(logger, "fetch device string, imsi:%s imei:%s phone_number:%s apn:%s", opt_val[0], opt_val[1], opt_val[2], opt_val[3]);
return htt_cnt_device;
}
int tfe_scan_port(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, uint16_t source, uint16_t dest)
{
int scan_ret = 0;
@@ -440,14 +538,12 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc
return hit_cnt_ip;
}
int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger)
int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
int scan_ret = 0;
int hit_cnt_ip = 0;
size_t n_hit_result = 0;
uint16_t opt_out_size;
char dest_subscribe_id[TFE_STRING_MAX] = {0};
char source_subscribe_id[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
@@ -457,13 +553,8 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
{
TFE_LOG_ERROR(logger, "fetch src sub id from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
// scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_SUB_ID, (unsigned char *)dest_subscribe_id, sizeof(dest_subscribe_id), &opt_out_size);
// if (scan_ret != 0)
// {
// TFE_LOG_ERROR(logger, "fetch dst sub id from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
// }
}
TFE_LOG_DEBUG(logger, "fetch src sub id:%s dst sub id:%s addr: %s", source_subscribe_id, dest_subscribe_id, stream->str_stream_info);
TFE_LOG_DEBUG(logger, "fetch src sub id:%s addr: %s", source_subscribe_id, stream->str_stream_info);
if (strlen(source_subscribe_id))
{
@@ -489,29 +580,5 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
}
}
if (strlen(dest_subscribe_id))
{
scan_ret = maat_scan_string(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_SUBSCRIBER_ID),
dest_subscribe_id, strlen(dest_subscribe_id),result + hit_cnt + hit_cnt_ip,
MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,&n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, Hit subid: %s scan ret: %d policy_id: %lld addr: %s",
dest_subscribe_id, scan_ret, result[hit_cnt + hit_cnt_ip], stream->str_stream_info);
hit_cnt_ip += n_hit_result;
}
else
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, NO hit subid: %s scan ret: %d addr: %s",
dest_subscribe_id, scan_ret, stream->str_stream_info);
}
scan_ret = maat_scan_not_logic(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_SUBSCRIBER_ID),
result + hit_cnt + hit_cnt_ip, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
}
}
return hit_cnt_ip;
}