TSG-17840 Proxy适配日志结构变更

This commit is contained in:
fengweihao
2023-12-08 18:12:53 +08:00
parent f1bf753679
commit b801ca9d3b
8 changed files with 551 additions and 146 deletions

View File

@@ -9,6 +9,13 @@ add_definitions(-D_GNU_SOURCE)
set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD 11)
set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD 11)
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
endif ()
if(NOT CMAKE_BUILD_TYPE) if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE RelWithDebInfo) set(CMAKE_BUILD_TYPE RelWithDebInfo)
endif() endif()

157
common/include/endianness.h Normal file
View File

@@ -0,0 +1,157 @@
#pragma once
#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__)
#include <endian.h>
#elif defined(__APPLE__) && defined(__MACH__)
#include <machine/endian.h>
#elif defined(BSD) || defined(_SYSTYPE_BSD)
#if defined(__OpenBSD__)
#include <machine/endian.h>
#else
#include <sys/endian.h>
#endif
#endif
#if defined(__BYTE_ORDER)
#if defined(__BIG_ENDIAN) && (__BYTE_ORDER == __BIG_ENDIAN)
#define BIGENDIAN
#elif defined(__LITTLE_ENDIAN) && (__BYTE_ORDER == __LITTLE_ENDIAN)
#define LITTLEENDIAN
#endif
#elif defined(_BYTE_ORDER)
#if defined(_BIG_ENDIAN) && (_BYTE_ORDER == _BIG_ENDIAN)
#define BIGENDIAN
#elif defined(_LITTLE_ENDIAN) && (_BYTE_ORDER == _LITTLE_ENDIAN)
#define LITTLEENDIAN
#endif
#elif defined(__BIG_ENDIAN__)
#define BIGENDIAN
#elif defined(__LITTLE_ENDIAN__)
#define LITTLEENDIAN
#else
#if defined(__ARMEL__) || \
defined(__THUMBEL__) || \
defined(__AARCH64EL__) || \
defined(_MIPSEL) || \
defined(__MIPSEL) || \
defined(__MIPSEL__) || \
defined(__ia64__) || defined(_IA64) || \
defined(__IA64__) || defined(__ia64) || \
defined(_M_IA64) || defined(__itanium__) || \
defined(i386) || defined(__i386__) || \
defined(__i486__) || defined(__i586__) || \
defined(__i686__) || defined(__i386) || \
defined(_M_IX86) || defined(_X86_) || \
defined(__THW_INTEL__) || defined(__I86__) || \
defined(__INTEL__) || \
defined(__x86_64) || defined(__x86_64__) || \
defined(__amd64__) || defined(__amd64) || \
defined(_M_X64) || \
defined(__bfin__) || defined(__BFIN__) || \
defined(bfin) || defined(BFIN)
#define LITTLEENDIAN
#elif defined(__m68k__) || defined(M68000) || \
defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \
defined(__sparc__) || defined(__sparc) || \
defined(__370__) || defined(__THW_370__) || \
defined(__s390__) || defined(__s390x__) || \
defined(__SYSC_ZARCH__)
#define BIGENDIAN
#elif defined(__arm__) || defined(__arm64) || defined(__thumb__) || \
defined(__TARGET_ARCH_ARM) || defined(__TARGET_ARCH_THUMB) || \
defined(__ARM_ARCH) || \
defined(_M_ARM) || defined(_M_ARM64)
#if defined(_WIN32) || defined(_WIN64) || \
defined(__WIN32__) || defined(__TOS_WIN__) || \
defined(__WINDOWS__)
#define LITTLEENDIAN
#else
#error "Cannot determine system endianness."
#endif
#endif
#endif
#if defined(BIGENDIAN)
// Try to use compiler intrinsics
#if defined(__INTEL_COMPILER) || defined(__ICC)
#define betole16(x) _bswap16(x)
#define betole32(x) _bswap(x)
#define betole64(x) _bswap64(x)
#elif defined(__GNUC__) // GCC and CLANG
#define betole16(x) __builtin_bswap16(x)
#define betole32(x) __builtin_bswap32(x)
#define betole64(x) __builtin_bswap64(x)
#elif defined(_MSC_VER) // MSVC
#include <stdlib.h>
#define betole16(x) _byteswap_ushort(x)
#define betole32(x) _byteswap_ulong(x)
#define betole64(x) _byteswap_uint64(x)
#else
#define FALLBACK_SWAP
#define betole16(x) swap_u16(x)
#define betole32(x) swap_u32(x)
#define betole64(x) swap_u64(x)
#endif
#define betole128(x) swap_u128(x)
#define betole256(x) swap_u256(x)
#else
#define betole16(x) (x)
#define betole32(x) (x)
#define betole64(x) (x)
#define betole128(x) (x)
#define betole256(x) (x)
#endif // BIGENDIAN
#if defined(BIGENDIAN)
#include <emmintrin.h>
#include <smmintrin.h>
#include <immintrin.h>
#include <tmmintrin.h>
inline __m128i swap_u128(__m128i value) {
const __m128i shuffle = _mm_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f);
return _mm_shuffle_epi8(value, shuffle);
}
inline __m256i swap_u256(__m256i value) {
const __m256i shuffle = _mm256_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f);
return _mm256_shuffle_epi8(value, shuffle);
}
#endif // BIGENDIAN
#if defined(FALLBACK_SWAP)
#include <stdint.h>
inline uint16_t swap_u16(uint16_t value)
{
return
((value & 0xFF00u) >> 8u) |
((value & 0x00FFu) << 8u);
}
inline uint32_t swap_u32(uint32_t value)
{
return
((value & 0xFF000000u) >> 24u) |
((value & 0x00FF0000u) >> 8u) |
((value & 0x0000FF00u) << 8u) |
((value & 0x000000FFu) << 24u);
}
inline uint64_t swap_u64(uint64_t value)
{
return
((value & 0xFF00000000000000u) >> 56u) |
((value & 0x00FF000000000000u) >> 40u) |
((value & 0x0000FF0000000000u) >> 24u) |
((value & 0x000000FF00000000u) >> 8u) |
((value & 0x00000000FF000000u) << 8u) |
((value & 0x0000000000FF0000u) << 24u) |
((value & 0x000000000000FF00u) << 40u) |
((value & 0x00000000000000FFu) << 56u);
}
#endif // FALLBACK_SWAP

View File

@@ -268,7 +268,7 @@ struct tfe_http_session
unsigned int session_id; unsigned int session_id;
short major_version; short major_version;
short minor_version; short minor_version;
time_t start_time; struct timeval start_time;
struct tfe_http_session_ops * ops; struct tfe_http_session_ops * ops;
struct tfe_http_half * req; struct tfe_http_half * req;

276
common/include/uuid_v4.h Normal file
View File

@@ -0,0 +1,276 @@
/*
MIT License
Copyright (c) 2018 Xavier "Crashoz" Launey
*/
#pragma once
#include <random>
#include <string>
#include <limits>
#include <iostream>
#include <sstream>
#include <cstdint>
#include <memory>
#include <emmintrin.h>
#include <smmintrin.h>
#include <immintrin.h>
#include "endianness.h"
namespace UUIDv4 {
/*
Converts a 128-bits unsigned int to an UUIDv4 string representation.
Uses SIMD via Intel's AVX2 instruction set.
*/
void inline m128itos(__m128i x, char* mem) {
// Expand each byte in x to two bytes in res
// i.e. 0x12345678 -> 0x0102030405060708
// Then translate each byte to its hex ascii representation
// i.e. 0x0102030405060708 -> 0x3132333435363738
const __m256i mask = _mm256_set1_epi8(0x0F);
const __m256i add = _mm256_set1_epi8(0x06);
const __m256i alpha_mask = _mm256_set1_epi8(0x10);
const __m256i alpha_offset = _mm256_set1_epi8(0x57);
__m256i a = _mm256_castsi128_si256(x);
__m256i as = _mm256_srli_epi64(a, 4);
__m256i lo = _mm256_unpacklo_epi8(as, a);
__m128i hi = _mm256_castsi256_si128(_mm256_unpackhi_epi8(as, a));
__m256i c = _mm256_inserti128_si256(lo, hi, 1);
__m256i d = _mm256_and_si256(c, mask);
__m256i alpha = _mm256_slli_epi64(_mm256_and_si256(_mm256_add_epi8(d, add), alpha_mask), 3);
__m256i offset = _mm256_blendv_epi8(_mm256_slli_epi64(add, 3), alpha_offset, alpha);
__m256i res = _mm256_add_epi8(d, offset);
// Add dashes between blocks as specified in RFC-4122
// 8-4-4-4-12
const __m256i dash_shuffle = _mm256_set_epi32(0x0b0a0908, 0x07060504, 0x80030201, 0x00808080, 0x0d0c800b, 0x0a090880, 0x07060504, 0x03020100);
const __m256i dash = _mm256_set_epi64x(0x0000000000000000ull, 0x2d000000002d0000ull, 0x00002d000000002d, 0x0000000000000000ull);
__m256i resd = _mm256_shuffle_epi8(res, dash_shuffle);
resd = _mm256_or_si256(resd, dash);
_mm256_storeu_si256((__m256i*)mem, betole256(resd));
*(uint16_t*)(mem+16) = betole16(_mm256_extract_epi16(res, 7));
*(uint32_t*)(mem+32) = betole32(_mm256_extract_epi32(res, 7));
}
/*
Converts an UUIDv4 string representation to a 128-bits unsigned int.
Uses SIMD via Intel's AVX2 instruction set.
*/
__m128i inline stom128i(const char* mem) {
// Remove dashes and pack hex ascii bytes in a 256-bits int
const __m256i dash_shuffle = _mm256_set_epi32(0x80808080, 0x0f0e0d0c, 0x0b0a0908, 0x06050403, 0x80800f0e, 0x0c0b0a09, 0x07060504, 0x03020100);
__m256i x = betole256(_mm256_loadu_si256((__m256i*)mem));
x = _mm256_shuffle_epi8(x, dash_shuffle);
x = _mm256_insert_epi16(x, betole16(*(uint16_t*)(mem+16)), 7);
x = _mm256_insert_epi32(x, betole32(*(uint32_t*)(mem+32)), 7);
// Build a mask to apply a different offset to alphas and digits
const __m256i sub = _mm256_set1_epi8(0x2F);
const __m256i mask = _mm256_set1_epi8(0x20);
const __m256i alpha_offset = _mm256_set1_epi8(0x28);
const __m256i digits_offset = _mm256_set1_epi8(0x01);
const __m256i unweave = _mm256_set_epi32(0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200, 0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200);
const __m256i shift = _mm256_set_epi32(0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004);
// Translate ascii bytes to their value
// i.e. 0x3132333435363738 -> 0x0102030405060708
// Shift hi-digits
// i.e. 0x0102030405060708 -> 0x1002300450067008
// Horizontal add
// i.e. 0x1002300450067008 -> 0x12345678
__m256i a = _mm256_sub_epi8(x, sub);
__m256i alpha = _mm256_slli_epi64(_mm256_and_si256(a, mask), 2);
__m256i sub_mask = _mm256_blendv_epi8(digits_offset, alpha_offset, alpha);
a = _mm256_sub_epi8(a, sub_mask);
a = _mm256_shuffle_epi8(a, unweave);
a = _mm256_sllv_epi32(a, shift);
a = _mm256_hadd_epi32(a, _mm256_setzero_si256());
a = _mm256_permute4x64_epi64(a, 0b00001000);
return _mm256_castsi256_si128(a);
}
/*
* UUIDv4 (random 128-bits) RFC-4122
*/
class UUID {
public:
UUID()
{}
UUID(const UUID &other) {
__m128i x = _mm_load_si128((__m128i*)other.data);
_mm_store_si128((__m128i*)data, x);
}
/* Builds a 128-bits UUID */
UUID(__m128i uuid) {
_mm_store_si128((__m128i*)data, uuid);
}
UUID(uint64_t x, uint64_t y) {
__m128i z = _mm_set_epi64x(x, y);
_mm_store_si128((__m128i*)data, z);
}
UUID(const uint8_t* bytes) {
__m128i x = _mm_loadu_si128((__m128i*)bytes);
_mm_store_si128((__m128i*)data, x);
}
/* Builds an UUID from a byte string (16 bytes long) */
explicit UUID(const std::string &bytes) {
__m128i x = betole128(_mm_loadu_si128((__m128i*)bytes.data()));
_mm_store_si128((__m128i*)data, x);
}
/* Static factory to parse an UUID from its string representation */
static UUID fromStrFactory(const std::string &s) {
return fromStrFactory(s.c_str());
}
static UUID fromStrFactory(const char* raw) {
return UUID(stom128i(raw));
}
void fromStr(const char* raw) {
_mm_store_si128((__m128i*)data, stom128i(raw));
}
UUID& operator=(const UUID &other) {
if (&other == this) {
return *this;
}
__m128i x = _mm_load_si128((__m128i*)other.data);
_mm_store_si128((__m128i*)data, x);
return *this;
}
friend bool operator==(const UUID &lhs, const UUID &rhs) {
__m128i x = _mm_load_si128((__m128i*)lhs.data);
__m128i y = _mm_load_si128((__m128i*)rhs.data);
__m128i neq = _mm_xor_si128(x, y);
return _mm_test_all_zeros(neq, neq);
}
friend bool operator<(const UUID &lhs, const UUID &rhs) {
// There are no trivial 128-bits comparisons in SSE/AVX
// It's faster to compare two uint64_t
uint64_t *x = (uint64_t*)lhs.data;
uint64_t *y = (uint64_t*)rhs.data;
return *x < *y || (*x == *y && *(x + 1) < *(y + 1));
}
friend bool operator!=(const UUID &lhs, const UUID &rhs) { return !(lhs == rhs); }
friend bool operator> (const UUID &lhs, const UUID &rhs) { return rhs < lhs; }
friend bool operator<=(const UUID &lhs, const UUID &rhs) { return !(lhs > rhs); }
friend bool operator>=(const UUID &lhs, const UUID &rhs) { return !(lhs < rhs); }
/* Serializes the uuid to a byte string (16 bytes) */
std::string bytes() const {
std::string mem;
bytes(mem);
return mem;
}
void bytes(std::string &out) const {
out.resize(sizeof(data));
bytes((char*)out.data());
}
void bytes(char* bytes) const {
__m128i x = betole128(_mm_load_si128((__m128i*)data));
_mm_storeu_si128((__m128i*)bytes, x);
}
/* Converts the uuid to its string representation */
std::string str() const {
std::string mem;
str(mem);
return mem;
}
void str(std::string &s) const {
s.resize(36);
str((char*)s.data());
}
void str(char *res) const {
__m128i x = _mm_load_si128((__m128i*)data);
m128itos(x, res);
}
friend std::ostream& operator<< (std::ostream& stream, const UUID& uuid) {
return stream << uuid.str();
}
friend std::istream& operator>> (std::istream& stream, UUID& uuid) {
std::string s;
stream >> s;
uuid = fromStrFactory(s);
return stream;
}
size_t hash() const {
const uint64_t a = *((uint64_t*)data);
const uint64_t b = *((uint64_t*)&data[8]);
return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
}
private:
alignas(128) uint8_t data[16];
};
/*
Generates UUIDv4 from a provided random generator (c++11 <random> module)
std::mt19937_64 is highly recommended as it has a SIMD implementation that
makes it very fast and it produces high quality randomness.
*/
template <typename RNG>
class UUIDGenerator {
public:
UUIDGenerator() : generator(new RNG(std::random_device()())), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
{}
UUIDGenerator(uint64_t seed) : generator(new RNG(seed)), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
{}
UUIDGenerator(RNG &gen) : generator(gen), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
{}
/* Generates a new UUID */
UUID getUUID() {
// The two masks set the uuid version (4) and variant (1)
const __m128i and_mask = _mm_set_epi64x(0xFFFFFFFFFFFFFF3Full, 0xFF0FFFFFFFFFFFFFull);
const __m128i or_mask = _mm_set_epi64x(0x0000000000000080ull, 0x0040000000000000ull);
__m128i n = _mm_set_epi64x(distribution(*generator), distribution(*generator));
__m128i uuid = _mm_or_si128(_mm_and_si128(n, and_mask), or_mask);
return UUID(uuid);
}
private:
std::shared_ptr<RNG> generator;
std::uniform_int_distribution<uint64_t> distribution;
};
}
namespace std {
template <> struct hash<UUIDv4::UUID>
{
size_t operator()(const UUIDv4::UUID & uuid) const
{
return uuid.hash();
}
};
}

View File

@@ -328,25 +328,20 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
common_obj = cJSON_CreateObject(); common_obj = cJSON_CreateObject();
cur_time = time(NULL); cur_time = time(NULL);
cJSON_AddNumberToObject(common_obj, "common_start_time", cur_time); cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", cur_time);
cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time); cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", cur_time);
cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]); cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "common_schema_type", "DoH"); cJSON_AddStringToObject(common_obj, "decoded_as", "DoH");
char opt_val[24] = { 0 }; char opt_val[24] = { 0 };
uint16_t opt_out_size; unsigned int common_direction=0; uint16_t opt_out_size;
struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream); struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
if (cmsg != NULL) if (cmsg != NULL)
{ {
int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
if (ret == 0) if (ret == 0)
{ {
cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val); cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size);
if (ret==0)
{
cJSON_AddNumberToObject(common_obj, "common_direction", common_direction); //0域内->域外1域外->域内描述的是CLIENT_IP信息
} }
} }
@@ -371,24 +366,22 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
switch (addr->addrtype) switch (addr->addrtype)
{ {
case TFE_ADDR_STREAM_TUPLE4_V4: case TFE_ADDR_STREAM_TUPLE4_V4:
cJSON_AddNumberToObject(common_obj, "common_address_type", 4); cJSON_AddNumberToObject(common_obj, "address_type", 4);
inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source)); cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest)); cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP");
break; break;
case TFE_ADDR_STREAM_TUPLE4_V6: case TFE_ADDR_STREAM_TUPLE4_V6:
cJSON_AddNumberToObject(common_obj, "common_address_type", 6); cJSON_AddNumberToObject(common_obj, "address_type", 6);
inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source)); cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest)); cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP");
break; break;
default: default:
break; break;
@@ -397,20 +390,21 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num)); tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num)); tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));
cJSON_AddNumberToObject(common_obj, "common_link_id", 0); cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);
cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id); cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id);
cJSON_AddNumberToObject(common_obj, "common_vsys_id", ctx->vsys_id); cJSON_AddNumberToObject(common_obj, "vsys_id", ctx->vsys_id);
cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id); cJSON_AddStringToObject(common_obj, "device_id", handle->device_id);
cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id); cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host); cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
if(handle->effective_device_tag) if(handle->effective_device_tag)
{ {
cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag); cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag);
} }
for (size_t i = 0; i < sizeof(req_fields) / sizeof(struct json_spec); i++) for (size_t i = 0; i < sizeof(req_fields) / sizeof(struct json_spec); i++)
@@ -432,19 +426,19 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
if (ctx->location_client) if (ctx->location_client)
{ {
cJSON_AddStringToObject(common_obj, "common_client_location", ctx->location_client); cJSON_AddStringToObject(common_obj, "client_geolocation", ctx->location_client);
} }
if (ctx->location_server) if (ctx->location_server)
{ {
cJSON_AddStringToObject(common_obj, "common_server_location", ctx->location_server); cJSON_AddStringToObject(common_obj, "server_geolocation", ctx->location_server);
} }
if (ctx->asn_client) if (ctx->asn_client)
{ {
cJSON_AddStringToObject(common_obj, "common_client_asn", ctx->asn_client); cJSON_AddStringToObject(common_obj, "client_asn", ctx->asn_client);
} }
if (ctx->asn_server) if (ctx->asn_server)
{ {
cJSON_AddStringToObject(common_obj, "common_server_asn", ctx->asn_server); cJSON_AddStringToObject(common_obj, "server_asn", ctx->asn_server);
} }
add_dns_info_to_log(common_obj, dns_info); add_dns_info_to_log(common_obj, dns_info);
@@ -462,11 +456,14 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
continue; continue;
} }
cJSON *proxy_rule_list=NULL;
int config_id[1]={0};
per_hit_obj = cJSON_Duplicate(common_obj, 1); per_hit_obj = cJSON_Duplicate(common_obj, 1);
cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", result[i].config_id); config_id[0]=result[i].config_id;
cJSON_AddNumberToObject(per_hit_obj, "common_service", result[i].service_id); proxy_rule_list = cJSON_CreateIntArray(config_id, 1);
cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE); cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list);
cJSON_AddStringToObject(per_hit_obj, "common_sub_action", "redirect"); cJSON_AddStringToObject(per_hit_obj, "proxy_action", "redirect");
log_payload = cJSON_PrintUnformatted(per_hit_obj); log_payload = cJSON_PrintUnformatted(per_hit_obj);

View File

@@ -43,6 +43,18 @@ enum _log_action //Bigger action number is prior.
__LG_ACTION_MAX __LG_ACTION_MAX
}; };
#define get_time_ms(tv) ((long long)(tv.tv_sec) * 1000 + (long long)(tv.tv_usec) / 1000)
#include "uuid_v4.h"
UUIDv4::UUIDGenerator<std::mt19937_64> uuidGenerator;
void get_http_body_uuid(char *uuid)
{
UUIDv4::UUID uid = uuidGenerator.getUUID();
uid.str(uuid);
return;
}
struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger) struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger)
{ {
struct tango_cache_parameter *log_file_upload_para=NULL; struct tango_cache_parameter *log_file_upload_para=NULL;
@@ -115,8 +127,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
char* log_payload=NULL; char* log_payload=NULL;
int kafka_status=0; int kafka_status=0;
int send_cnt=0; int send_cnt=0;
int tmp=0; struct timeval cur_time;
time_t cur_time;
char src_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0}; char src_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};
char dst_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0}; char dst_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};
@@ -145,14 +156,15 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
} }
common_obj=cJSON_CreateObject(); common_obj=cJSON_CreateObject();
cur_time = time(NULL); gettimeofday(&cur_time, NULL);
cJSON_AddNumberToObject(common_obj, "common_start_time", http->start_time); cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time); cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]); cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
cJSON_AddStringToObject(common_obj, "common_schema_type", "HTTP"); cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");
unsigned int common_direction=0, category_id_val[64]={0};
unsigned int category_id_val[64]={0};
char opt_val[24]={0}; uint16_t opt_out_size; char opt_val[24]={0}; uint16_t opt_out_size;
struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream); struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
if (cmsg!=NULL) if (cmsg!=NULL)
@@ -160,17 +172,12 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size); int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size);
if (ret==0) if (ret==0)
{ {
cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val); cJSON_AddStringToObject(common_obj, "session_id", opt_val);
}
ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size);
if (ret==0)
{
cJSON_AddNumberToObject(common_obj, "common_direction", common_direction); //69域内->域外73域外->域内描述的是CLIENT_IP信息
} }
ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val)); ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val));
if (ret>0) if (ret>0)
{ {
cJSON_AddItemToObject(common_obj, "common_service_category", cJSON_CreateIntArray((const int*)category_id_val, ret)); cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret));
} }
} }
@@ -189,30 +196,29 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
struct tfe_http_resp_spec resp_spec=http->resp->resp_spec; struct tfe_http_resp_spec resp_spec=http->resp->resp_spec;
asprintf(&response_line, "HTTP/%d.%d %d OK", http->major_version, http->minor_version, resp_spec.resp_code); asprintf(&response_line, "HTTP/%d.%d %d OK", http->major_version, http->minor_version, resp_spec.resp_code);
cJSON_AddStringToObject(common_obj, "http_response_line", response_line); cJSON_AddStringToObject(common_obj, "http_response_line", response_line);
cJSON_AddNumberToObject(common_obj, "http_status_code", resp_spec.resp_code);
free(response_line); free(response_line);
} }
switch(addr->addrtype) switch(addr->addrtype)
{ {
case TFE_ADDR_STREAM_TUPLE4_V4: case TFE_ADDR_STREAM_TUPLE4_V4:
cJSON_AddNumberToObject(common_obj, "common_address_type", 4); cJSON_AddNumberToObject(common_obj, "address_type", 4);
inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source)); cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest)); cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP");
break; break;
case TFE_ADDR_STREAM_TUPLE4_V6: case TFE_ADDR_STREAM_TUPLE4_V6:
cJSON_AddNumberToObject(common_obj, "common_address_type", 6); cJSON_AddNumberToObject(common_obj, "address_type", 6);
inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str));
inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str));
cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source)); cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source));
cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest)); cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest));
cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP");
break; break;
default: default:
break; break;
@@ -230,19 +236,20 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
s2c_byte_num = log_msg->s2c_byte_num; s2c_byte_num = log_msg->s2c_byte_num;
} }
cJSON_AddNumberToObject(common_obj, "common_link_id", 0); cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str); cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);
cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id); cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id);
cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id); cJSON_AddStringToObject(common_obj, "device_id", handle->device_id);
cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id); cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num); cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num);
cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host); cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host);
cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
if(handle->effective_device_tag) if(handle->effective_device_tag)
{ {
cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag); cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag);
} }
for(size_t i=0;i<sizeof(req_fields)/sizeof(struct json_spec);i++) for(size_t i=0;i<sizeof(req_fields)/sizeof(struct json_spec);i++)
@@ -262,11 +269,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
} }
} }
char log_file_upload_req_path[TFE_STRING_MAX]={0}, cont_type_whole[TFE_STRING_MAX]={0}; #define FILE_CHUNK_UUID_LEN 40
char log_file_upload_resp_path[TFE_STRING_MAX]={0}; char uuid[FILE_CHUNK_UUID_LEN]={0};
memset(log_file_upload_req_path, 0, sizeof(log_file_upload_req_path));
memset(log_file_upload_resp_path, 0, sizeof(log_file_upload_resp_path));
memset(cont_type_whole, 0, sizeof(cont_type_whole));
for(size_t i=0; i<log_msg->result_num; i++) for(size_t i=0; i<log_msg->result_num; i++)
{ {
@@ -280,81 +284,34 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
continue; continue;
} }
struct tango_cache_meta_put meta;
char* log_file_key=NULL;;
const char* cont_type_val;
if(log_msg->req_body!=NULL) if(log_msg->req_body!=NULL)
{ {
if(log_file_upload_req_path[0] != '\0') if(uuid[0] != '\0')
{ {
cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path); cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
} }
else else
{ {
memset(&meta, 0, sizeof(meta)); get_http_body_uuid(uuid);
asprintf(&log_file_key, "%s.reqbody", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
meta.url=log_file_key;
cont_type_val=tfe_http_std_field_read(http->req, TFE_HTTP_CONT_TYPE);
if(cont_type_val!=NULL)
{
snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
meta.std_hdr[0]=cont_type_whole;
}
meta.user_log_name=1;
tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
log_msg->req_body,
&meta,
log_file_upload_req_path, sizeof(log_file_upload_req_path));
if(tmp==0)
{
cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed.");
}
free(log_file_key);
} }
} }
if(log_msg->resp_body!=NULL) if(log_msg->resp_body!=NULL)
{ {
if(log_file_upload_resp_path[0] != '\0') if(uuid[0] != '\0')
{ {
cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path); cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
} }
else else
{ {
memset(&meta, 0, sizeof(meta)); get_http_body_uuid(uuid);
asprintf(&log_file_key, "%s.respbody", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
meta.url=log_file_key;
cont_type_val=tfe_http_std_field_read(http->resp, TFE_HTTP_CONT_TYPE);
if(cont_type_val!=NULL)
{
snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
meta.std_hdr[0]=cont_type_whole;
}
meta.user_log_name=1;
tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
log_msg->resp_body,
&meta,
log_file_upload_resp_path, sizeof(log_file_upload_resp_path));
if(tmp==0)
{
cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path);
}
else
{
TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed.");
}
free(log_file_key);
} }
} }
} }
for(size_t i=0; i<log_msg->result_num; i++) for(size_t i=0; i<log_msg->result_num; i++)
{ {
TFE_LOG_DEBUG(handle->local_logger, "URL: %s, policy_id: %lld, service: %d, do_log:%d", TFE_LOG_DEBUG(handle->local_logger, "URL: %s, policy_id: %lld, service: %d, do_log:%d",
http->req->req_spec.url, http->req->req_spec.url,
log_msg->result[i].config_id, log_msg->result[i].config_id,
@@ -366,27 +323,38 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
continue; continue;
} }
cJSON *proxy_rule_list=NULL;
int config_id[1]={0};
per_hit_obj=cJSON_Duplicate(common_obj, 1); per_hit_obj=cJSON_Duplicate(common_obj, 1);
cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", log_msg->result[i].config_id); config_id[0]=log_msg->result[i].config_id;
cJSON_AddNumberToObject(per_hit_obj, "common_service", log_msg->result[i].service_id); proxy_rule_list = cJSON_CreateIntArray(config_id, 1);
cJSON_AddNumberToObject(per_hit_obj, "common_vsys_id", log_msg->result[i].vsys_id); cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list);
cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE); cJSON_AddNumberToObject(per_hit_obj, "vsys_id", log_msg->result[i].vsys_id);
if(log_msg->result[i].action == LG_ACTION_MANIPULATE) if(log_msg->result[i].action == LG_ACTION_MANIPULATE)
{ {
cJSON_AddStringToObject(per_hit_obj, "common_sub_action", manipulate_action_map[log_msg->action]); cJSON_AddStringToObject(per_hit_obj, "proxy_action", manipulate_action_map[log_msg->action]);
cJSON_AddNumberToObject(per_hit_obj, "http_action_file_size", log_msg->inject_sz); cJSON_AddNumberToObject(per_hit_obj, "http_action_file_size", log_msg->inject_sz);
} }
else else
{ {
cJSON_AddStringToObject(per_hit_obj, "common_sub_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]); cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
} }
if(log_msg->location_client) if(log_msg->location_client)
{ {
cJSON_AddStringToObject(per_hit_obj, "common_client_location", log_msg->location_client); cJSON_AddStringToObject(per_hit_obj, "client_geolocation", log_msg->location_client);
} }
if(log_msg->location_server) if(log_msg->location_server)
{ {
cJSON_AddStringToObject(per_hit_obj, "common_server_location", log_msg->location_server); cJSON_AddStringToObject(per_hit_obj, "server_geolocation", log_msg->location_server);
}
if(log_msg->asn_client)
{
cJSON_AddStringToObject(common_obj, "client_asn", log_msg->asn_client);
}
if (log_msg->asn_server)
{
cJSON_AddStringToObject(common_obj, "server_asn", log_msg->asn_server);
} }
log_payload = cJSON_PrintUnformatted(per_hit_obj); log_payload = cJSON_PrintUnformatted(per_hit_obj);

View File

@@ -1136,7 +1136,7 @@ struct http_session_private * hs_private_create(struct http_connection_private *
__hs_private->hs_public.req = hf_private_req != NULL ? to_hf_public(hf_private_req) : NULL; __hs_private->hs_public.req = hf_private_req != NULL ? to_hf_public(hf_private_req) : NULL;
__hs_private->hs_public.resp = hf_private_req != NULL ? to_hf_public(hf_private_resp) : NULL; __hs_private->hs_public.resp = hf_private_req != NULL ? to_hf_public(hf_private_resp) : NULL;
__hs_private->hs_public.session_id = hc_private->session_id_counter++; __hs_private->hs_public.session_id = hc_private->session_id_counter++;
__hs_private->hs_public.start_time=time(NULL); gettimeofday(&(__hs_private->hs_public.start_time), NULL);
/* HS-PRIVATE*/ /* HS-PRIVATE*/
__hs_private->hc_private = hc_private; __hs_private->hc_private = hc_private;

View File

@@ -1270,7 +1270,7 @@ static int tfe_half_session_init(struct tfe_h2_session *h2_session, int32_t stre
struct tfe_h2_half_private *req = h2_session->req; struct tfe_h2_half_private *req = h2_session->req;
tfe_session->ops = &http2_session_ops; tfe_session->ops = &http2_session_ops;
tfe_session->req = &req->half_public; tfe_session->req = &req->half_public;
tfe_session->start_time=time(NULL); gettimeofday(&tfe_session->start_time, NULL);
tfe_session->session_id = stream_id; tfe_session->session_id = stream_id;
} }