gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-22316 Manipulation支持Device相关策略的扫描与日志发送, TSG-22051 Manipulation日志发送Library相关字段, TSG-22256 Manipulation日志发送Incoming Link ID和Outgoing Link ID等字段

Browse Source
This commit is contained in:
fengweihao
2024-08-30 10:25:16 +08:00
parent 9e2b08ab53
commit 80eed59202
11 changed files with 529 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -9,6 +9,7 @@
#include "kafka.h"
#include "mpack.h"
#include "tsg_proxy_logger.h"
#include "tfe_scan.h"
struct json_spec
{
@@ -110,7 +111,7 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se
return instance;
}
int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
int tfe_get_format_host(cJSON *common_obj, const char *req_spec_host)
{
unsigned int port;
char *format_host=ALLOC(char, strlen(req_spec_host)+1);
@@ -121,27 +122,82 @@ int proxy_add_host_to_object(cJSON *common_obj, const char *req_spec_host)
return 0;
}
int tags_line_to_json_array(cJSON *per_hit_obj, const char *tags_key, char *opt_val)
int tfe_get_integer_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
if(per_hit_obj == NULL || tags_key == NULL || opt_val == NULL)
uint16_t opt_out_size = 0;
unsigned int integer = 0;
int ret = tfe_cmsg_get_value(cmsg, type, (unsigned char *)&integer, sizeof(integer), &opt_out_size);
if(ret == 0 && type == TFE_CMSG_COMMON_DIRECTION)
{
return 0;
const char *direction = (integer == 69) ? "Outbound" : "Inbound";
cJSON_AddStringToObject(common_obj, keyword, direction);
}
if (ret == 0 && type != TFE_CMSG_COMMON_DIRECTION)
{
cJSON_AddNumberToObject(common_obj, keyword, integer);
}
return 0;
}
int tfe_get_string_by_cmsg(cJSON *common_obj, struct tfe_cmsg * cmsg, enum tfe_cmsg_tlv_type type, const char *keyword)
{
char opt_val[128]={0};
uint16_t opt_out_size = 0;
int ret=tfe_cmsg_get_value(cmsg, type, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && opt_out_size > 0)
{
cJSON_AddStringToObject(common_obj, keyword, opt_val);
}
return 0;
}
size_t tfe_get_c2s_byte_num(const struct tfe_stream *stream, size_t c2s_byte_num)
{
size_t rewrite_c2s_byte_num = 0;
int ret = tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &rewrite_c2s_byte_num, sizeof(rewrite_c2s_byte_num));
if(ret != 0)
{
rewrite_c2s_byte_num = c2s_byte_num == 0 ? rewrite_c2s_byte_num : c2s_byte_num;
}
return rewrite_c2s_byte_num;
}
size_t tfe_get_s2c_byte_num(const struct tfe_stream *stream, size_t s2c_byte_num)
{
size_t ret=0, rewrite_s2c_byte_num =0;
ret = tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &rewrite_s2c_byte_num, sizeof(rewrite_s2c_byte_num));
if(ret !=0)
{
rewrite_s2c_byte_num = s2c_byte_num == 0 ? rewrite_s2c_byte_num : s2c_byte_num;
}
return rewrite_s2c_byte_num;
}
int tfe_upload_http_body(struct proxy_logger* handle, cJSON *common_obj, struct evbuffer *http_body, char *uuid, const char *keyword)
{
size_t datalen=0;
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, keyword, uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, http_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, keyword, uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload %s failed.", keyword);
}
}
char *opt_val_tmp = strdup(opt_val);
cJSON *tags_array = cJSON_CreateArray();
char *token = strtok(opt_val_tmp, ",");
while (token != NULL)
{
while (*token == ' ') token++;
cJSON_AddItemToArray(tags_array, cJSON_CreateString(token));
token = strtok(NULL, ",");
}
cJSON_AddItemToObject(per_hit_obj, tags_key, tags_array);
FREE(&opt_val_tmp)
return 1;
return 0;
}
int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
@@ -184,24 +240,27 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
gettimeofday(&cur_time, NULL);
cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
char source_subscribe_id[64]={0};
char opt_val[24]={0}; uint16_t opt_out_size;
struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg!=NULL)
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg != NULL)
{
int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_SUB_ID, (unsigned char *)source_subscribe_id, sizeof(source_subscribe_id), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "subscriber_id", source_subscribe_id);
}
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_STREAM_TRACE_ID, "session_id");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_SUB_ID, "subscriber_id");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMSI_STR, "imsi");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_IMEI_STR, "imei");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_PHONE_NUM_STR, "phone_number");
tfe_get_string_by_cmsg(common_obj, cmsg, TFE_CMSG_SRC_APN_STR, "apn");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_INCOMING_LINK_ID, "in_link_id");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_OUTGOING_LINK_ID, "out_link_id");
tfe_get_integer_by_cmsg(common_obj, cmsg, TFE_CMSG_COMMON_DIRECTION, "direction");
}
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_SRC_IP_TAGS_IDS_STR, "client_ip_tags");
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_DST_IP_TAGS_IDS_STR, "server_ip_tags");
tfe_get_library_tags(log_msg->stream, common_obj, TFE_CMSG_FQDN_TAGS_IDS_STR, "server_fqdn_tags");
if (http->req)
{
char *request_line=NULL;
@@ -245,30 +304,16 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
break;
}
size_t ret=0, c2s_byte_num = 0, s2c_byte_num =0;
ret = tfe_stream_info_get(log_msg->stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
if(ret != 0)
{
c2s_byte_num = log_msg->c2s_byte_num == 0 ? c2s_byte_num : log_msg->c2s_byte_num;
}
ret = tfe_stream_info_get(log_msg->stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));
if(ret !=0)
{
s2c_byte_num = log_msg->s2c_byte_num == 0 ? s2c_byte_num : log_msg->s2c_byte_num;
}
cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
cJSON_AddStringToObject(common_obj, "ip_protocol", "tcp");
cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "sled_ip", tfe_get_sled_ip());
cJSON_AddNumberToObject(common_obj, "t_vsys_id", tfe_get_vsys_id());
cJSON_AddStringToObject(common_obj, "device_id", tfe_get_device_id());
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddNumberToObject(common_obj, "sent_bytes", tfe_get_c2s_byte_num(log_msg->stream, log_msg->c2s_byte_num));
cJSON_AddNumberToObject(common_obj, "received_bytes", tfe_get_s2c_byte_num(log_msg->stream, log_msg->s2c_byte_num));
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
proxy_add_host_to_object(common_obj, http->req->req_spec.host);
tfe_get_format_host(common_obj, http->req->req_spec.host);
if (tfe_get_device_tag())
{
@@ -293,8 +338,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
}
#define FILE_CHUNK_UUID_LEN 40
char uuid[FILE_CHUNK_UUID_LEN]={0};
size_t datalen=0;
char http_req_uuid[FILE_CHUNK_UUID_LEN]={0};
char http_resp_uuid[FILE_CHUNK_UUID_LEN]={0};
for(size_t i=0; i<log_msg->result_num; i++)
{
@@ -302,43 +347,11 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
if(log_msg->req_body!=NULL)
{
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, log_msg->req_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed.");
}
}
tfe_upload_http_body(handle, common_obj, log_msg->req_body, http_req_uuid, "http_request_body");
}
if(log_msg->resp_body!=NULL)
{
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
else
{
get_http_body_uuid(uuid);
datalen=file_bucket_upload_once(handle, uuid, log_msg->resp_body);
if(datalen>0)
{
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed.");
}
}
tfe_upload_http_body(handle, common_obj, log_msg->resp_body, http_resp_uuid, "http_response_body");
}
}
@@ -395,29 +408,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
{
cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
}
if (cmsg!=NULL)
{
char opt_val[128]={0}; uint16_t opt_out_size;
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)64, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "client_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)65, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "server_ip_tags", opt_val);
}
memset(opt_val, 0, sizeof(opt_val));
ret = tfe_cmsg_get_value(cmsg, (enum tfe_cmsg_tlv_type)63, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0 && strlen(opt_val) > 0)
{
tags_line_to_json_array(per_hit_obj, "server_fqdn_tags", opt_val);
}
}
log_payload = cJSON_PrintUnformatted(per_hit_obj);