gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-22316 Manipulation支持Device相关策略的扫描与日志发送, TSG-22051 Manipulation日志发送Library相关字段, TSG-22256 Manipulation日志发送Incoming Link ID和Outgoing Link ID等字段

Browse Source
This commit is contained in:
fengweihao
2024-08-30 10:25:16 +08:00
parent 9e2b08ab53
commit 80eed59202
11 changed files with 529 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
common/include/tfe_resource.h
View File

@@ -1,5 +1,18 @@
#pragma once
enum category_type
{
CATEGORY_TYPE_UNKNOWN = 0,
CATEGORY_TYPE_GEOIP,
CATEGORY_TYPE_CONTRY_CODE,
CATEGORY_TYPE_ASN,
CATEGORY_TYPE_WEBSITE_CATEGORY,
CATEGORY_TYPE_INTERNET_SERVICE,
CATEGORY_TYPE_SECURITY_THREAT,
CATEGORY_TYPE_RISK,
CATEGORY_TYPE_MAX
};
struct app_id_dict
{
int ref_cnt;
@@ -9,7 +22,19 @@ struct app_id_dict
pthread_mutex_t lock;
};
struct library_tag_ctx
{
int ref_cnt;
int tag_id;
char *tag_key;
char *tag_value;
enum category_type category;
pthread_mutex_t lock;
};
void app_id_dict_free(struct app_id_dict *app_dict);
void library_tag_free(struct library_tag_ctx *library_tags);
enum scan_common_table
{
@@ -24,6 +49,11 @@ enum scan_common_table
PXY_CTRL_IP_PROTOCOL,
PXY_CTRL_SUBSCRIBER_ID,
PXY_CTRL_APP_ID_DICT,
PXY_CTRL_LIBRARY_TAG,
PXY_CTRL_IMSI,
PXY_CTRL_APN,
PXY_CTRL_PHONE_NUMBER,
PXY_CTRL_GTP_IMEI,
__SCAN_COMMON_TABLE_MAX
};

4
common/include/tfe_scan.h
View File

@@ -1,5 +1,6 @@
#pragma once
#include <cjson/cJSON.h>
#include <tfe_stream.h>
#define MAX_SCAN_RESULT 16
@@ -10,3 +11,6 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt,
int tfe_scan_ipv4_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);
int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, struct ipaddr sapp_addr);
int tfe_scan_port(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, uint16_t source, uint16_t dest);
int tfe_scan_device(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger);
int tfe_get_library_tags(const struct tfe_stream *stream, cJSON *common_obj, tfe_cmsg_tlv_type tlv_type, const char *tag_key);

4
common/src/packet.cpp
View File

@@ -42,8 +42,8 @@
* Static API
******************************************************************************/
static const char *ethproto_tostring(uint16_t proto);
static const char *ipproto_tostring(uint16_t proto);
static const char *ethproto_tostring(uint16_t proto)__attribute__((unused));
static const char *ipproto_tostring(uint16_t proto)__attribute__((unused));
static inline const char *ldbc_method_tostring(enum ldbc_method method);
static inline const char *layer_type_tostring(enum layer_type type);

2
common/src/tfe_ctrl_packet.cpp
View File

@@ -64,7 +64,7 @@ int tags_ids_cmsg_maps[] = {
[INDEX_FQDN_TAGS_IDS] = TFE_CMSG_FQDN_TAGS_IDS_STR,
};
char *tags_ids_cmsg_name_maps[] = {
const char *tags_ids_cmsg_name_maps[] = {
[INDEX_SRC_IP_TAGS_IDS] = "TFE_CMSG_SRC_IP_TAGS_IDS_STR",
[INDEX_DST_IP_TAGS_IDS] = "TFE_CMSG_DST_IP_TAGS_IDS_STR",
[INDEX_FQDN_TAGS_IDS] = "TFE_CMSG_FQDN_TAGS_IDS_STR",

113
common/src/tfe_resource.cpp
View File

@@ -319,6 +319,110 @@ void app_dict_table_dup_cb(int table_id, void **to, void **from, long argl, void
return;
}
int get_category_type_str2idx(const char *category)
{
size_t i = 0;
const char *category_name[] = {"unknown", "geoip_city", "geoip_country", "geoip_asn", "website_classfication", "internet_service", "security_threat", "compliance_risk"};
for (i = 0; i < sizeof(category_name) / sizeof(const char *); i++)
{
if (0 == strcasecmp(category, category_name[i]))
break;
}
return i;
}
void library_tag_new_cb(const char *table_name, int table_id, const char* key, const char* table_line, void **ad, long argl, void* argp)
{
int ret=0;
size_t offset=0, len=0;
char category[256]={0};
struct library_tag_ctx *library_tags = ALLOC(struct library_tag_ctx, 1);
ret = maat_helper_read_column(table_line, 1, &offset, &len);
if(ret >= 0)
{
char *tag_id_str=ALLOC(char, len+1);
memcpy(tag_id_str, table_line+offset, len);
library_tags->tag_id=atoi(tag_id_str);
FREE(&tag_id_str);
}
ret = maat_helper_read_column(table_line, 3, &offset, &len);
if(ret >= 0)
{
memcpy(category, table_line+offset, len);
library_tags->category=(enum category_type)get_category_type_str2idx(category);
}
ret = maat_helper_read_column(table_line, 4, &offset, &len);
if(ret >= 0)
{
library_tags->tag_key=ALLOC(char, len+1);
memcpy(library_tags->tag_key, table_line+offset, len);
}
ret = maat_helper_read_column(table_line, 5, &offset, &len);
if(ret >= 0)
{
library_tags->tag_value=ALLOC(char, len+1);
memcpy(library_tags->tag_value, table_line+offset, len);
}
library_tags->ref_cnt=1;
pthread_mutex_init(&(library_tags->lock), NULL);
*ad=library_tags;
return;
}
void library_tag_free_cb(int table_id, void **ad, long argl, void* argp)
{
if(*ad==NULL)
{
return;
}
struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*ad);
pthread_mutex_lock(&(library_tags->lock));
library_tags->ref_cnt--;
if(library_tags->ref_cnt>0)
{
pthread_mutex_unlock(&(library_tags->lock));
return;
}
pthread_mutex_unlock(&(library_tags->lock));
pthread_mutex_destroy(&(library_tags->lock));
if(library_tags->tag_key)
{
FREE(&library_tags->tag_key);
}
if(library_tags->tag_value)
{
FREE(&library_tags->tag_value);
}
FREE(&library_tags);
*ad=NULL;
return;
}
void library_tag_dup_cb(int table_id, void **to, void **from, long argl, void* argp)
{
struct library_tag_ctx *library_tags=(struct library_tag_ctx *)(*from);
pthread_mutex_lock(&(library_tags->lock));
library_tags->ref_cnt++;
pthread_mutex_unlock(&(library_tags->lock));
*to=library_tags;
return;
}
void library_tag_free(struct library_tag_ctx *library_tags)
{
library_tag_free_cb(0, (void **)&library_tags, 0, NULL);
}
static int maat_common_table_init()
{
const char * table_name[__SCAN_COMMON_TABLE_MAX];
@@ -333,6 +437,11 @@ static int maat_common_table_init()
table_name[PXY_CTRL_IP_PROTOCOL] = "ATTR_IP_PROTOCOL";
table_name[PXY_CTRL_SUBSCRIBER_ID] = "ATTR_SUBSCRIBER_ID";
table_name[PXY_CTRL_APP_ID_DICT] = "APP_ID_DICT";
table_name[PXY_CTRL_LIBRARY_TAG] = "LIBRARY_TAG";
table_name[PXY_CTRL_IMSI]="ATTR_GTP_IMSI";
table_name[PXY_CTRL_APN]="ATTR_GTP_APN";
table_name[PXY_CTRL_PHONE_NUMBER]="ATTR_GTP_PHONE_NUMBER";
table_name[PXY_CTRL_GTP_IMEI]="ATTR_GTP_IMEI";
for (int i = 0; i < __SCAN_COMMON_TABLE_MAX; i++)
{
@@ -344,7 +453,9 @@ static int maat_common_table_init()
}
}
maat_plugin_table_ex_schema_register(maat_handle, "APP_ID_DICT", app_dict_table_new_cb, app_dict_table_free_cb, app_dict_table_dup_cb, 0, NULL);
return 0;
maat_plugin_table_ex_schema_register(maat_handle, "LIBRARY_TAG", library_tag_new_cb, library_tag_free_cb, library_tag_dup_cb, 0, NULL);
return 0;
}
int tfe_env_init()

213
common/src/tfe_scan.cpp
View File

@@ -114,34 +114,79 @@ int tfe_scan_internal_exteral_by_last_group(const struct tfe_stream *stream, lon
return hit_cnt_group;
}
/**for Provisional compilation definitions**/
#define TFE_CMSG_FQDN_TAGS_LOG_STR 63 /*"Category Name:Search Engines, Cloude Provider:aws, CDN Provider:aws, category_name: ab"*/
#define TFE_CMSG_SRC_TAGS_LOG_STR 64 /*"administrative_area:Singapore, country:Singapore", CDN Provider:Light CDN*/
#define TFE_CMSG_DST_TAGS_LOG_STR 65 /*"ASN:55967, CDN Provider:Light CDN, country:Hong Kong"*/
int tfe_get_entry_tags(const struct tfe_stream * stream, enum tfe_cmsg_tlv_type tlv_type, char *opt_val, long long *tag_id_array)
{
int n_tag_ids = 0;
uint16_t opt_out_size = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
{
return 0;
}
int ret = tfe_cmsg_get_value(cmsg, tlv_type, (unsigned char *)opt_val, 128, &opt_out_size);
if(ret == 0 && opt_out_size > 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
}
return n_tag_ids;
}
int tfe_get_library_tags(const struct tfe_stream *stream, cJSON *common_obj, tfe_cmsg_tlv_type tlv_type, const char *tag_key)
{
char opt_val[128]={0};
long long tag_id_array[128]={0};
int n_tag_ids = tfe_get_entry_tags(stream, tlv_type, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return 0;
}
char value[128]={0};
cJSON *tags_array = cJSON_CreateArray();
for(int i = 0; i < n_tag_ids; i++)
{
struct library_tag_ctx *library_tag =(struct library_tag_ctx *)maat_plugin_table_get_ex_data(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_LIBRARY_TAG), (const char *)&tag_id_array[i], sizeof(long long));
if(library_tag != NULL)
{
if(library_tag->category == CATEGORY_TYPE_ASN && tlv_type == TFE_CMSG_SRC_IP_TAGS_IDS_STR)
{
cJSON_AddNumberToObject(common_obj, "client_asn", atol(library_tag->tag_value));
}
if(library_tag->category == CATEGORY_TYPE_ASN && tlv_type == TFE_CMSG_DST_IP_TAGS_IDS_STR)
{
cJSON_AddNumberToObject(common_obj, "server_asn", atol(library_tag->tag_value));
}
memset(value, 0, sizeof(value));
snprintf(value, sizeof(value), "%s:%s", library_tag->tag_key, library_tag->tag_value);
cJSON_AddItemToArray(tags_array, cJSON_CreateString(value));
}
library_tag_free(library_tag);
}
cJSON_AddItemToObject(common_obj, tag_key, tags_array);
return 0;
}
int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
size_t n_hit_result = 0;
long long tag_id_array[128]={0};
int scan_ret = 0, hit_cnt_ip = 0, n_tag_ids = 0;
char opt_val[128]={0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_SRC_IP_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
char opt_val[128]={0}; uint16_t opt_out_size = 0;
int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_SRC_IP_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
}
TFE_LOG_DEBUG(logger, "fetch src ip tags: %s", opt_val);
struct maat_hit_group hit_group;
for (int i = 0; i < n_tag_ids; i++)
{
@@ -166,18 +211,16 @@ int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct
}
}
n_tag_ids=0;
memset(opt_val, 0, sizeof(opt_val));
memset(tag_id_array, 0, sizeof(tag_id_array));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_DST_IP_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_DST_IP_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_ip;
}
return hit_cnt_ip;
}
TFE_LOG_DEBUG(logger, "fetch dst ip tags: %s", opt_val);
for (int i = 0; i < n_tag_ids; i++)
{
memset(&hit_group, 0, sizeof(hit_group));
@@ -204,27 +247,17 @@ int tfe_scan_ip_tags(const struct tfe_stream *stream, long long *result, struct
int tfe_scan_fqdn_tags(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, int table_id, void *logger)
{
char opt_val[128]={0};
long long tag_id_array[128]={0};
int scan_ret = 0, hit_cnt_fqdn = 0, n_tag_ids = 0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
n_tag_ids = tfe_get_entry_tags(stream, (enum tfe_cmsg_tlv_type)TFE_CMSG_FQDN_TAGS_IDS_STR, opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_fqdn;
}
char opt_val[128]={0}; uint16_t opt_out_size = 0;
int ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)TFE_CMSG_FQDN_TAGS_IDS_STR, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if(ret == 0)
{
n_tag_ids = tfe_get_tags_id_array(opt_val, tag_id_array);
if(n_tag_ids == 0)
{
return hit_cnt_fqdn;
}
}
TFE_LOG_DEBUG(logger, "fetch fqdn tags: %s", opt_val);
struct maat_hit_group hit_group;
for (int i = 0; i < n_tag_ids; i++)
{
@@ -241,7 +274,7 @@ int tfe_scan_fqdn_tags(const struct tfe_stream *stream, long long *result, struc
TFE_LOG_INFO(logger, "Scan Fqdn TAGS, NO hit scan ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
}
return hit_cnt_fqdn;
return hit_cnt_fqdn;
}
int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt, long long app_id, int table_id)
@@ -273,6 +306,71 @@ int tfe_scan_app_id(long long *result, struct maat_state *scan_mid, int hit_cnt,
return hit_app_id;
}
int tfe_scan_value_by_cmsg(const struct tfe_stream *stream, enum tfe_cmsg_tlv_type tlv_type, long long *result, struct maat_state *scan_mid, int hit_cnt, int table_id, char *opt_val, void *logger)
{
uint16_t opt_out_size = 0;
int hit_cnt_string=0;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if(cmsg == NULL)
{
return hit_cnt_string;
}
int ret = tfe_cmsg_get_value(cmsg, tlv_type, (unsigned char *)opt_val, 128, &opt_out_size);
if(ret == 0 && opt_out_size > 0)
{
size_t n_hit_result=0;
int scan_ret = maat_scan_string(tfe_get_maat_handle(), table_id, opt_val, strlen(opt_val), result+hit_cnt+hit_cnt_string, MAX_SCAN_RESULT-hit_cnt-hit_cnt_string,
&n_hit_result,scan_mid);
if(scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_string+=n_hit_result;
}
scan_ret = maat_scan_not_logic(tfe_get_maat_handle(), table_id, result+hit_cnt+hit_cnt_string, MAX_SCAN_RESULT-hit_cnt-hit_cnt_string, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_string+=n_hit_result;
}
}
return hit_cnt_string;
}
int tfe_scan_device(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
char opt_val[4][128]={0};
int scan_ret = 0, htt_cnt_device = 0;
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_IMSI_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_IMSI), opt_val[0], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_IMEI_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_GTP_IMEI), opt_val[1], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_PHONE_NUM_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_PHONE_NUMBER), opt_val[2], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
scan_ret = tfe_scan_value_by_cmsg(stream, TFE_CMSG_SRC_APN_STR, result, scan_mid, hit_cnt, tfe_bussiness_tableid_get(PXY_CTRL_APN), opt_val[3], logger);
if(scan_ret > 0)
{
htt_cnt_device += scan_ret;
}
TFE_LOG_DEBUG(logger, "fetch device string, imsi:%s imei:%s phone_number:%s apn:%s", opt_val[0], opt_val[1], opt_val[2], opt_val[3]);
return htt_cnt_device;
}
int tfe_scan_port(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, uint16_t source, uint16_t dest)
{
int scan_ret = 0;
@@ -440,14 +538,12 @@ int tfe_scan_ipv6_addr(const struct tfe_stream *stream, long long *result, struc
return hit_cnt_ip;
}
int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid,
int hit_cnt, void *logger)
int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, struct maat_state *scan_mid, int hit_cnt, void *logger)
{
int scan_ret = 0;
int hit_cnt_ip = 0;
size_t n_hit_result = 0;
uint16_t opt_out_size;
char dest_subscribe_id[TFE_STRING_MAX] = {0};
char source_subscribe_id[TFE_STRING_MAX] = {0};
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
@@ -457,13 +553,8 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
{
TFE_LOG_ERROR(logger, "fetch src sub id from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
}
// scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_SUB_ID, (unsigned char *)dest_subscribe_id, sizeof(dest_subscribe_id), &opt_out_size);
// if (scan_ret != 0)
// {
// TFE_LOG_ERROR(logger, "fetch dst sub id from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info);
// }
}
TFE_LOG_DEBUG(logger, "fetch src sub id:%s dst sub id:%s addr: %s", source_subscribe_id, dest_subscribe_id, stream->str_stream_info);
TFE_LOG_DEBUG(logger, "fetch src sub id:%s addr: %s", source_subscribe_id, stream->str_stream_info);
if (strlen(source_subscribe_id))
{
@@ -489,29 +580,5 @@ int tfe_scan_subscribe_id(const struct tfe_stream *stream, long long *result, st
}
}
if (strlen(dest_subscribe_id))
{
scan_ret = maat_scan_string(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_SUBSCRIBER_ID),
dest_subscribe_id, strlen(dest_subscribe_id),result + hit_cnt + hit_cnt_ip,
MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip,&n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, Hit subid: %s scan ret: %d policy_id: %lld addr: %s",
dest_subscribe_id, scan_ret, result[hit_cnt + hit_cnt_ip], stream->str_stream_info);
hit_cnt_ip += n_hit_result;
}
else
{
TFE_LOG_INFO(logger, "Scan dst TSG_OBJ_SUBSCRIBER_ID, NO hit subid: %s scan ret: %d addr: %s",
dest_subscribe_id, scan_ret, stream->str_stream_info);
}
scan_ret = maat_scan_not_logic(tfe_get_maat_handle(), tfe_bussiness_tableid_get(PXY_CTRL_SUBSCRIBER_ID),
result + hit_cnt + hit_cnt_ip, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip, &n_hit_result, scan_mid);
if (scan_ret == MAAT_SCAN_HIT)
{
hit_cnt_ip += n_hit_result;
}
}
return hit_cnt_ip;
}

1
common/test/test_mpack.cpp
View File

@@ -38,7 +38,6 @@ void build_mpack_data(char **data, size_t *size)
char src_imei[256] = "25762917001364";
char src_phone_number[256] = "623335886431";
char src_apn[256] = "www.lytest.com";
uint64_t src_ip_tags_ids[] = {18446744073709551615ULL, 324137, 324129};
uint64_t dst_ip_tags_ids[] = {324119, 18446744073709551615ULL, 324137, 324129};
uint64_t fqdn_tags_ids[] = {324109, 324105, 18446744073709551615ULL, 324137, 324129};
uint8_t ja3_fingerprint[32] = {0};

6
plugin/business/doh/src/doh.cpp
View File

@@ -350,6 +350,12 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
hit_cnt += scan_ret;
}
scan_ret = tfe_scan_device(stream, result, ctx->scan_mid, hit_cnt, g_doh_conf->local_logger);
if(scan_ret > 0)
{
hit_cnt += scan_ret;
}
// scan qname
scan_ret = maat_scan_string(g_doh_conf->maat, g_doh_conf->tables[TYPE_QNAME].id, qname, strlen(qname),
result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid);

99
plugin/business/doh/src/logger.cpp
View File

@@ -1,3 +1,4 @@
#include "tfe_scan.h"
#include "logger.h"
#include "kafka.h"
@@ -292,7 +293,7 @@ int doh_kafka_init(const char *profile, struct doh_conf *conf)
return 0;
}
int doh_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
int doh_get_format_host(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
@@ -303,27 +304,35 @@ int doh_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
return 0;
}
int doh_tags_line_to_json_array(cJSON *per_hit_obj, const char *tags_key, char *opt_val)
int doh_get_integer_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
if(per_hit_obj == NULL || tags_key == NULL || opt_val == NULL)
uint16_t opt_out_size = 0;
unsigned int integer = 0;
int ret = tfe_cmsg_get_value(cmsg, type, (unsigned char *)&integer, sizeof(integer), &opt_out_size);
if(ret == 0 && type == TFE_CMSG_COMMON_DIRECTION)
{
return 0;
const char *direction = (integer == 69) ? "Outbound" : "Inbound";
cJSON_AddStringToObject(common_obj, keyword, direction);
}
char *opt_val_tmp = strdup(opt_val);
cJSON *tags_array = cJSON_CreateArray();
char *token = strtok(opt_val_tmp, ",");
while (token != NULL)
if (ret == 0 && type != TFE_CMSG_COMMON_DIRECTION)
{
while (*token == ' ') token++;
cJSON_AddItemToArray(tags_array, cJSON_CreateString(token));
token = strtok(NULL, ",");
}
cJSON_AddItemToObject(per_hit_obj, tags_key, tags_array);
cJSON_AddNumberToObject(common_obj, keyword, integer);
}
return 0;
}
FREE(&opt_val_tmp)
return 1;
int doh_get_string_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
char opt_val[128]={0};
uint16_t opt_out_size = 0;
int ret=tfe_cmsg_get_value(cmsg, type, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && opt_out_size > 0)
{
cJSON_AddStringToObject(common_obj, keyword, opt_val);
}
return 0;
}
int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, const struct tfe_stream *stream, struct doh_ctx *ctx)
@@ -361,24 +370,26 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "decoded_as", "DoH");
char opt_val[24]={0};
char source_subscribe_id[64]={0};
uint16_t opt_out_size;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL)
{
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0)
{
cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_SUB_ID, (unsigned char *)source_subscribe_id, sizeof(source_subscribe_id), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id);
}
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_STREAM_TRACE_ID, "session_id");
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_SUB_ID, "subscriber_id");
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMSI_STR, "imsi");
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMEI_STR, "imei");
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_PHONE_NUM_STR, "phone_number");
doh_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_APN_STR, "apn");
doh_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_INCOMING_LINK_ID, "in_link_id");
doh_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_OUTGOING_LINK_ID, "out_link_id");
doh_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_COMMON_DIRECTION, "direction");
}
tfe_get_library_tags(stream, common_obj, TFE_CMSG_SRC_IP_TAGS_IDS_STR, "client_ip_tags");
tfe_get_library_tags(stream, common_obj, TFE_CMSG_DST_IP_TAGS_IDS_STR, "server_ip_tags");
tfe_get_library_tags(stream, common_obj, TFE_CMSG_FQDN_TAGS_IDS_STR, "server_fqdn_tags");
if (http->req)
{
char *request_line = NULL;
@@ -420,7 +431,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
default:
break;
}
size_t ret=0, c2s_byte_num = 0, s2c_byte_num = 0;
size_t c2s_byte_num = 0, s2c_byte_num = 0;
tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));
@@ -434,7 +445,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url);
doh_add_host_to_object(common_obj, http->req->req_spec.host);
doh_get_format_host(common_obj, http->req->req_spec.host);
if(tfe_get_device_tag())
{
@@ -458,30 +469,6 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
}
}
if (cmsg!=NULL)
{
char opt_val[128]={0}; uint16_t opt_out_size=0;
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)64, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
doh_tags_line_to_json_array(common_obj, "client_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)65, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
doh_tags_line_to_json_array(common_obj, "server_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)63, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
doh_tags_line_to_json_array(common_obj, "server_fqdn_tags", opt_val);
}
}
add_dns_info_to_log(common_obj, dns_info);
for (size_t i = 0; i < result_num; i++)
{

206
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -9,6 +9,7 @@
#include "kafka.h"
#include "mpack.h"
#include "tsg_proxy_logger.h"
#include "tfe_scan.h"
struct json_spec
{
@@ -110,7 +111,7 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se
return instance;
}
int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
int tfe_get_format_host(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
@@ -121,27 +122,82 @@ int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
return 0;
}
int tags_line_to_json_array(cJSON *per_hit_obj, const char *tags_key, char *opt_val)
int tfe_get_integer_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
if(per_hit_obj == NULL || tags_key == NULL || opt_val == NULL)
uint16_t opt_out_size = 0;
unsigned int integer = 0;
int ret = tfe_cmsg_get_value(cmsg, type, (unsigned char *)&integer, sizeof(integer), &opt_out_size);
if(ret == 0 && type == TFE_CMSG_COMMON_DIRECTION)
{
return 0;
const char *direction = (integer == 69) ? "Outbound" : "Inbound";
cJSON_AddStringToObject(common_obj, keyword, direction);
}
if (ret == 0 && type != TFE_CMSG_COMMON_DIRECTION)
{
cJSON_AddNumberToObject(common_obj, keyword, integer);
}
return 0;
}
int tfe_get_string_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
char opt_val[128]={0};
uint16_t opt_out_size = 0;
int ret=tfe_cmsg_get_value(cmsg, type, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && opt_out_size > 0)
{
cJSON_AddStringToObject(common_obj, keyword, opt_val);
}
return 0;
}
size_t tfe_get_c2s_byte_num(const struct tfe_stream *stream, size_t c2s_byte_num)
{
size_t rewrite_c2s_byte_num = 0;
int ret = tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &rewrite_c2s_byte_num, sizeof(rewrite_c2s_byte_num));
if(ret != 0)
{
rewrite_c2s_byte_num = c2s_byte_num == 0 ? rewrite_c2s_byte_num : c2s_byte_num;
}
return rewrite_c2s_byte_num;
}
size_t tfe_get_s2c_byte_num(const struct tfe_stream *stream, size_t s2c_byte_num)
{
size_t ret=0, rewrite_s2c_byte_num =0;
ret = tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &rewrite_s2c_byte_num, sizeof(rewrite_s2c_byte_num));
if(ret !=0)
{
rewrite_s2c_byte_num = s2c_byte_num == 0 ? rewrite_s2c_byte_num : s2c_byte_num;
}
return rewrite_s2c_byte_num;
}
int tfe_upload_http_body(struct proxy_logger* handle, cJSON *common_obj, struct evbuffer *http_body, char *uuid, const char *keyword)
{
size_t datalen=0;
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, keyword, uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, http_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, keyword, uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload %s failed.", keyword);
}
}
char *opt_val_tmp = strdup(opt_val);
cJSON *tags_array = cJSON_CreateArray();
char *token = strtok(opt_val_tmp, ",");
while (token != NULL)
{
while (*token == ' ') token++;
cJSON_AddItemToArray(tags_array, cJSON_CreateString(token));
token = strtok(NULL, ",");
}
cJSON_AddItemToObject(per_hit_obj, tags_key, tags_array);
FREE(&opt_val_tmp)
return 1;
return 0;
}
int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
@@ -185,23 +241,26 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
char source_subscribe_id[64]={0};
char opt_val[24]={0}; uint16_t opt_out_size;
struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg!=NULL)
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg != NULL)
{
int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_SUB_ID, (unsigned char *)source_subscribe_id, sizeof(source_subscribe_id), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id);
}
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_STREAM_TRACE_ID, "session_id");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_SUB_ID, "subscriber_id");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMSI_STR, "imsi");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMEI_STR, "imei");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_PHONE_NUM_STR, "phone_number");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_APN_STR, "apn");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_INCOMING_LINK_ID, "in_link_id");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_OUTGOING_LINK_ID, "out_link_id");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_COMMON_DIRECTION, "direction");
}
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_SRC_IP_TAGS_IDS_STR, "client_ip_tags");
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_DST_IP_TAGS_IDS_STR, "server_ip_tags");
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_FQDN_TAGS_IDS_STR, "server_fqdn_tags");
if (http->req)
{
char *request_line=NULL;
@@ -245,30 +304,16 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
break;
}
size_t ret=0, c2s_byte_num = 0, s2c_byte_num =0;
ret = tfe_stream_info_get(log_msg->stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
if(ret != 0)
{
c2s_byte_num = log_msg->c2s_byte_num == 0 ? c2s_byte_num : log_msg->c2s_byte_num;
}
ret = tfe_stream_info_get(log_msg->stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));
if(ret !=0)
{
s2c_byte_num = log_msg->s2c_byte_num == 0 ? s2c_byte_num : log_msg->s2c_byte_num;
}
cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp");
cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "sled_ip", tfe_get_sled_ip());
cJSON_AddNumberToObject(common_obj, "t_vsys_id", tfe_get_vsys_id());
cJSON_AddStringToObject(common_obj, "device_id", tfe_get_device_id());
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddNumberToObject(common_obj, "sent_bytes", tfe_get_c2s_byte_num(log_msg->stream, log_msg->c2s_byte_num));
cJSON_AddNumberToObject(common_obj, "received_bytes", tfe_get_s2c_byte_num(log_msg->stream, log_msg->s2c_byte_num));
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
proxy_add_host_to_object(common_obj, http->req->req_spec.host);
tfe_get_format_host(common_obj, http->req->req_spec.host);
if (tfe_get_device_tag())
{
@@ -293,8 +338,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
}
#define FILE_CHUNK_UUID_LEN 40
char uuid[FILE_CHUNK_UUID_LEN]={0};
size_t datalen=0;
char http_req_uuid[FILE_CHUNK_UUID_LEN]={0};
char http_resp_uuid[FILE_CHUNK_UUID_LEN]={0};
for(size_t i=0; i<log_msg->result_num; i++)
{
@@ -302,43 +347,11 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
if(log_msg->req_body!=NULL)
{
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, log_msg->req_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed.");
}
}
tfe_upload_http_body(handle, common_obj, log_msg->req_body, http_req_uuid, "http_request_body");
}
if(log_msg->resp_body!=NULL)
{
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, log_msg->resp_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed.");
}
}
tfe_upload_http_body(handle, common_obj, log_msg->resp_body, http_resp_uuid, "http_response_body");
}
}
@@ -395,29 +408,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{
cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
}
if (cmsg!=NULL)
{
char opt_val[128]={0}; uint16_t opt_out_size;
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)64, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "client_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)65, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "server_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)63, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "server_fqdn_tags", opt_val);
}
}
log_payload = cJSON_PrintUnformatted(per_hit_obj);

91
resource/pangu/table_info.conf
View File

@@ -496,5 +496,96 @@
"table_name": "ATTR_IP_PROTOCOL",
"table_type": "virtual",
"physical_table": "TSG_IP_PROTOCOL"
},
{
"table_id": 50,
"table_name": "LIBRARY_TAG",
"table_type": "plugin",
"valid_column": 6,
"custom": {
"key": 1,
"key_type": "integer",
"key_len": 8
}
},
{
"table_id":51,
"table_name":"TSG_OBJ_IMSI",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":52,
"table_name":"TSG_OBJ_PHONE_NUMBER",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":53,
"table_name":"TSG_OBJ_APN",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":54,
"table_name":"TSG_OBJ_IMEI",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":55,
"table_name":"ATTR_GTP_IMSI",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IMSI"
},
{
"table_id":56,
"table_name":"ATTR_GTP_PHONE_NUMBER",
"table_type":"virtual",
"physical_table": "TSG_OBJ_PHONE_NUMBER"
},
{
"table_id":57,
"table_name":"ATTR_GTP_APN",
"table_type":"virtual",
"physical_table": "TSG_OBJ_APN"
},
{
"table_id":58,
"table_name":"ATTR_GTP_IMEI",
"table_type":"virtual",
"physical_table": "TSG_OBJ_IMEI"
}
]