gfwleak/tango-tfe
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-17840 Proxy适配日志结构变更

Browse Source
This commit is contained in:
fengweihao
2023-12-08 18:12:53 +08:00
parent f1bf753679
commit b801ca9d3b
8 changed files with 551 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
plugin/business/tsg-http/src/tsg_logger.cpp
View File

@@ -43,6 +43,18 @@ enum _log_action //Bigger action number is prior.
__LG_ACTION_MAX
};
#define get_time_ms(tv) ((long long)(tv.tv_sec) * 1000 + (long long)(tv.tv_usec) / 1000)
#include "uuid_v4.h"
UUIDv4::UUIDGenerator<std::mt19937_64> uuidGenerator;
void get_http_body_uuid(char *uuid)
{
UUIDv4::UUID uid = uuidGenerator.getUUID();
uid.str(uuid);
return;
}
struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger)
{
struct tango_cache_parameter *log_file_upload_para=NULL;
@@ -115,8 +127,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
char* log_payload=NULL;
int kafka_status=0;
int send_cnt=0;
int tmp=0;
time_t cur_time;
struct timeval cur_time;
char src_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};
char dst_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};
@@ -145,14 +156,15 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
}
common_obj=cJSON_CreateObject();
cur_time = time(NULL);
gettimeofday(&cur_time, NULL);
cJSON_AddNumberToObject(common_obj, "common_start_time", http->start_time);
cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time);
cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "common_schema_type", "HTTP");
cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
unsigned int common_direction=0, category_id_val[64]={0};
unsigned int category_id_val[64]={0};
char opt_val[24]={0}; uint16_t opt_out_size;
struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg!=NULL)
@@ -160,17 +172,12 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size);
if (ret==0)
{
cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val);
cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size);
if (ret==0)
{
cJSON_AddNumberToObject(common_obj, "common_direction", common_direction); //69：域内->域外，73：域外->域内，描述的是CLIENT_IP信息
}
ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val));
if (ret>0)
{
cJSON_AddItemToObject(common_obj, "common_service_category", cJSON_CreateIntArray((const int*)category_id_val, ret));
cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret));
}
}
@@ -189,30 +196,29 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
struct tfe_http_resp_spec resp_spec=http->resp->resp_spec;
asprintf(&response_line, "HTTP/%d.%d %d OK", http->major_version, http->minor_version, resp_spec.resp_code);
cJSON_AddStringToObject(common_obj, "http_response_line", response_line);
cJSON_AddNumberToObject(common_obj, "http_status_code", resp_spec.resp_code);
free(response_line);
}
switch(addr->addrtype)
{
case TFE_ADDR_STREAM_TUPLE4_V4:
cJSON_AddNumberToObject(common_obj, "common_address_type", 4);
cJSON_AddNumberToObject(common_obj, "address_type", 4);
inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP");
cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source));
cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest));
break;
case TFE_ADDR_STREAM_TUPLE4_V6:
cJSON_AddNumberToObject(common_obj, "common_address_type", 6);
cJSON_AddNumberToObject(common_obj, "address_type", 6);
inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP");
cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source));
cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest));
break;
default:
break;
@@ -230,19 +236,20 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
s2c_byte_num = log_msg->s2c_byte_num;
}
cJSON_AddNumberToObject(common_obj, "common_link_id", 0);
cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double
cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str);
cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id);
cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id);
cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id);
cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num);
cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);
cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id);
cJSON_AddStringToObject(common_obj, "device_id", handle->device_id);
cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
if(handle->effective_device_tag)
{
cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag);
cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag);
}
for(size_t i=0;i<sizeof(req_fields)/sizeof(struct json_spec);i++)
@@ -262,11 +269,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
}
}
char log_file_upload_req_path[TFE_STRING_MAX]={0}, cont_type_whole[TFE_STRING_MAX]={0};
char log_file_upload_resp_path[TFE_STRING_MAX]={0};
memset(log_file_upload_req_path, 0, sizeof(log_file_upload_req_path));
memset(log_file_upload_resp_path, 0, sizeof(log_file_upload_resp_path));
memset(cont_type_whole, 0, sizeof(cont_type_whole));
#define FILE_CHUNK_UUID_LEN 40
char uuid[FILE_CHUNK_UUID_LEN]={0};
for(size_t i=0; i<log_msg->result_num; i++)
{
@@ -280,81 +284,34 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
continue;
}
struct tango_cache_meta_put meta;
char* log_file_key=NULL;;
const char* cont_type_val;
if(log_msg->req_body!=NULL)
{
if(log_file_upload_req_path[0] != '\0')
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path);
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
else
{
memset(&meta, 0, sizeof(meta));
asprintf(&log_file_key, "%s.reqbody", http->req->req_spec.url);
meta.url=log_file_key;
cont_type_val=tfe_http_std_field_read(http->req, TFE_HTTP_CONT_TYPE);
if(cont_type_val!=NULL)
{
snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
meta.std_hdr[0]=cont_type_whole;
}
meta.user_log_name=1;
tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
log_msg->req_body,
&meta,
log_file_upload_req_path, sizeof(log_file_upload_req_path));
if(tmp==0)
{
cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed.");
}
free(log_file_key);
get_http_body_uuid(uuid);
cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
}
}
if(log_msg->resp_body!=NULL)
{
if(log_file_upload_resp_path[0] != '\0')
if(uuid[0] != '\0')
{
cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path);
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
else
{
memset(&meta, 0, sizeof(meta));
asprintf(&log_file_key, "%s.respbody", http->req->req_spec.url);
meta.url=log_file_key;
cont_type_val=tfe_http_std_field_read(http->resp, TFE_HTTP_CONT_TYPE);
if(cont_type_val!=NULL)
{
snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
meta.std_hdr[0]=cont_type_whole;
}
meta.user_log_name=1;
tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
log_msg->resp_body,
&meta,
log_file_upload_resp_path, sizeof(log_file_upload_resp_path));
if(tmp==0)
{
cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed.");
}
free(log_file_key);
get_http_body_uuid(uuid);
cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
}
}
}
for(size_t i=0; i<log_msg->result_num; i++)
{
TFE_LOG_DEBUG(handle->local_logger, "URL: %s, policy_id: %lld, service: %d, do_log:%d",
http->req->req_spec.url,
log_msg->result[i].config_id,
@@ -366,27 +323,38 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
continue;
}
cJSON *proxy_rule_list=NULL;
int config_id[1]={0};
per_hit_obj=cJSON_Duplicate(common_obj, 1);
cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", log_msg->result[i].config_id);
cJSON_AddNumberToObject(per_hit_obj, "common_service", log_msg->result[i].service_id);
cJSON_AddNumberToObject(per_hit_obj, "common_vsys_id", log_msg->result[i].vsys_id);
cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE);
config_id[0]=log_msg->result[i].config_id;
proxy_rule_list = cJSON_CreateIntArray(config_id, 1);
cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list);
cJSON_AddNumberToObject(per_hit_obj, "vsys_id", log_msg->result[i].vsys_id);
if(log_msg->result[i].action == LG_ACTION_MANIPULATE)
{
cJSON_AddStringToObject(per_hit_obj, "common_sub_action", manipulate_action_map[log_msg->action]);
cJSON_AddStringToObject(per_hit_obj, "proxy_action", manipulate_action_map[log_msg->action]);
cJSON_AddNumberToObject(per_hit_obj, "http_action_file_size", log_msg->inject_sz);
}
else
{
cJSON_AddStringToObject(per_hit_obj, "common_sub_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
}
if(log_msg->location_client)
{
cJSON_AddStringToObject(per_hit_obj, "common_client_location", log_msg->location_client);
cJSON_AddStringToObject(per_hit_obj, "client_geolocation", log_msg->location_client);
}
if(log_msg->location_server)
{
cJSON_AddStringToObject(per_hit_obj, "common_server_location", log_msg->location_server);
cJSON_AddStringToObject(per_hit_obj, "server_geolocation", log_msg->location_server);
}
if(log_msg->asn_client)
{
cJSON_AddStringToObject(common_obj, "client_asn", log_msg->asn_client);
}
if (log_msg->asn_server)
{
cJSON_AddStringToObject(common_obj, "server_asn", log_msg->asn_server);
}
log_payload = cJSON_PrintUnformatted(per_hit_obj);