TSG-17840 Proxy适配日志结构变更

2023-12-08 18:12:53 +08:00
parent f1bf753679
commit b801ca9d3b
8 changed files with 551 additions and 146 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,6 +9,13 @@ add_definitions(-D_GNU_SOURCE)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_C_STANDARD 11)

+if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
+  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd")
+endif ()
+
 if(NOT CMAKE_BUILD_TYPE)
 	set(CMAKE_BUILD_TYPE RelWithDebInfo)
 endif()
--- a/common/include/endianness.h
+++ b/common/include/endianness.h
@@ -0,0 +1,157 @@
+#pragma once
+
+#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__)
+  #include <endian.h>
+#elif defined(__APPLE__) && defined(__MACH__)
+  #include <machine/endian.h>
+#elif defined(BSD) || defined(_SYSTYPE_BSD)
+  #if defined(__OpenBSD__)
+    #include <machine/endian.h>
+  #else
+    #include <sys/endian.h>
+  #endif
+#endif
+
+#if defined(__BYTE_ORDER)
+  #if defined(__BIG_ENDIAN) && (__BYTE_ORDER == __BIG_ENDIAN)
+    #define BIGENDIAN
+  #elif defined(__LITTLE_ENDIAN) && (__BYTE_ORDER == __LITTLE_ENDIAN)
+    #define LITTLEENDIAN
+  #endif
+#elif defined(_BYTE_ORDER)
+  #if defined(_BIG_ENDIAN) && (_BYTE_ORDER == _BIG_ENDIAN)
+    #define BIGENDIAN
+  #elif defined(_LITTLE_ENDIAN) && (_BYTE_ORDER == _LITTLE_ENDIAN)
+    #define LITTLEENDIAN
+  #endif
+#elif defined(__BIG_ENDIAN__)
+  #define BIGENDIAN
+#elif defined(__LITTLE_ENDIAN__)
+  #define LITTLEENDIAN
+#else
+  #if defined(__ARMEL__) || \
+      defined(__THUMBEL__) || \
+      defined(__AARCH64EL__) || \
+      defined(_MIPSEL) || \
+      defined(__MIPSEL) || \
+      defined(__MIPSEL__) || \
+      defined(__ia64__) || defined(_IA64) || \
+      defined(__IA64__) || defined(__ia64) || \
+      defined(_M_IA64) || defined(__itanium__) || \
+      defined(i386) || defined(__i386__) || \
+      defined(__i486__) || defined(__i586__) || \
+      defined(__i686__) || defined(__i386) || \
+      defined(_M_IX86) || defined(_X86_) || \
+      defined(__THW_INTEL__) || defined(__I86__) || \
+      defined(__INTEL__) || \
+      defined(__x86_64) || defined(__x86_64__) || \
+      defined(__amd64__) || defined(__amd64) || \
+      defined(_M_X64) || \
+      defined(__bfin__) || defined(__BFIN__) || \
+      defined(bfin) || defined(BFIN)
+
+      #define LITTLEENDIAN
+  #elif defined(__m68k__) || defined(M68000) || \
+        defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \
+        defined(__sparc__) || defined(__sparc) || \
+        defined(__370__) || defined(__THW_370__) || \
+        defined(__s390__) || defined(__s390x__) || \
+        defined(__SYSC_ZARCH__)
+
+      #define BIGENDIAN
+
+  #elif defined(__arm__) || defined(__arm64) || defined(__thumb__) || \
+        defined(__TARGET_ARCH_ARM) || defined(__TARGET_ARCH_THUMB) || \
+        defined(__ARM_ARCH) || \
+        defined(_M_ARM) || defined(_M_ARM64)
+
+      #if defined(_WIN32) || defined(_WIN64) || \
+          defined(__WIN32__) || defined(__TOS_WIN__) || \
+          defined(__WINDOWS__)
+
+        #define LITTLEENDIAN
+
+      #else
+        #error "Cannot determine system endianness."
+      #endif
+  #endif
+#endif
+
+
+#if defined(BIGENDIAN)
+  // Try to use compiler intrinsics
+  #if defined(__INTEL_COMPILER) || defined(__ICC)
+    #define betole16(x) _bswap16(x)
+    #define betole32(x) _bswap(x)
+    #define betole64(x) _bswap64(x)
+  #elif defined(__GNUC__) // GCC and CLANG
+    #define betole16(x) __builtin_bswap16(x)
+    #define betole32(x) __builtin_bswap32(x)
+    #define betole64(x) __builtin_bswap64(x)
+  #elif defined(_MSC_VER) // MSVC
+    #include <stdlib.h>
+    #define betole16(x) _byteswap_ushort(x)
+    #define betole32(x) _byteswap_ulong(x)
+    #define betole64(x) _byteswap_uint64(x)
+  #else
+    #define FALLBACK_SWAP
+    #define betole16(x) swap_u16(x)
+    #define betole32(x) swap_u32(x)
+    #define betole64(x) swap_u64(x)
+  #endif
+  #define betole128(x) swap_u128(x)
+  #define betole256(x) swap_u256(x)
+#else
+  #define betole16(x) (x)
+  #define betole32(x) (x)
+  #define betole64(x) (x)
+  #define betole128(x) (x)
+  #define betole256(x) (x)
+#endif // BIGENDIAN
+
+#if defined(BIGENDIAN)
+  #include <emmintrin.h>
+  #include <smmintrin.h>
+  #include <immintrin.h>
+  #include <tmmintrin.h>
+
+  inline __m128i swap_u128(__m128i value) {
+    const __m128i shuffle = _mm_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f);
+    return _mm_shuffle_epi8(value, shuffle);
+  }
+
+  inline __m256i swap_u256(__m256i value) {
+    const __m256i shuffle = _mm256_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f);
+    return _mm256_shuffle_epi8(value, shuffle);
+  }
+#endif // BIGENDIAN
+
+#if defined(FALLBACK_SWAP)
+  #include <stdint.h>
+  inline uint16_t swap_u16(uint16_t value)
+  {
+    return
+        ((value & 0xFF00u) >>  8u) |
+        ((value & 0x00FFu) <<  8u);
+  }
+  inline uint32_t swap_u32(uint32_t value)
+  {
+    return
+        ((value & 0xFF000000u) >> 24u) |
+        ((value & 0x00FF0000u) >>  8u) |
+        ((value & 0x0000FF00u) <<  8u) |
+        ((value & 0x000000FFu) << 24u);
+  }
+  inline uint64_t swap_u64(uint64_t value)
+  {
+    return
+        ((value & 0xFF00000000000000u) >> 56u) |
+        ((value & 0x00FF000000000000u) >> 40u) |
+        ((value & 0x0000FF0000000000u) >> 24u) |
+        ((value & 0x000000FF00000000u) >>  8u) |
+        ((value & 0x00000000FF000000u) <<  8u) |
+        ((value & 0x0000000000FF0000u) << 24u) |
+        ((value & 0x000000000000FF00u) << 40u) |
+        ((value & 0x00000000000000FFu) << 56u);
+  }
+#endif // FALLBACK_SWAP
--- a/common/include/tfe_http.h
+++ b/common/include/tfe_http.h
@@ -268,7 +268,7 @@ struct tfe_http_session
 	unsigned int session_id;
 	short major_version;
 	short minor_version;
-    time_t start_time;
+    struct timeval start_time;

 	struct tfe_http_session_ops * ops;
 	struct tfe_http_half * req;
--- a/common/include/uuid_v4.h
+++ b/common/include/uuid_v4.h
@@ -0,0 +1,276 @@
+/*
+MIT License
+
+Copyright (c) 2018 Xavier "Crashoz" Launey
+*/
+
+#pragma once
+
+#include <random>
+#include <string>
+#include <limits>
+#include <iostream>
+#include <sstream>
+#include <cstdint>
+#include <memory>
+
+#include <emmintrin.h>
+#include <smmintrin.h>
+#include <immintrin.h>
+
+#include "endianness.h"
+
+namespace UUIDv4 {
+
+/*
+  Converts a 128-bits unsigned int to an UUIDv4 string representation.
+  Uses SIMD via Intel's AVX2 instruction set.
+ */
+void inline m128itos(__m128i x, char* mem) {
+  // Expand each byte in x to two bytes in res
+  // i.e. 0x12345678 -> 0x0102030405060708
+  // Then translate each byte to its hex ascii representation
+  // i.e. 0x0102030405060708 -> 0x3132333435363738
+  const __m256i mask = _mm256_set1_epi8(0x0F);
+  const __m256i add = _mm256_set1_epi8(0x06);
+  const __m256i alpha_mask = _mm256_set1_epi8(0x10);
+  const __m256i alpha_offset = _mm256_set1_epi8(0x57);
+
+  __m256i a = _mm256_castsi128_si256(x);
+  __m256i as = _mm256_srli_epi64(a, 4);
+  __m256i lo = _mm256_unpacklo_epi8(as, a);
+  __m128i hi = _mm256_castsi256_si128(_mm256_unpackhi_epi8(as, a));
+  __m256i c =  _mm256_inserti128_si256(lo, hi, 1);
+  __m256i d = _mm256_and_si256(c, mask);
+  __m256i alpha = _mm256_slli_epi64(_mm256_and_si256(_mm256_add_epi8(d, add), alpha_mask), 3);
+  __m256i offset = _mm256_blendv_epi8(_mm256_slli_epi64(add, 3), alpha_offset, alpha);
+  __m256i res = _mm256_add_epi8(d, offset);
+
+  // Add dashes between blocks as specified in RFC-4122
+  // 8-4-4-4-12
+  const __m256i dash_shuffle = _mm256_set_epi32(0x0b0a0908, 0x07060504, 0x80030201, 0x00808080, 0x0d0c800b, 0x0a090880, 0x07060504, 0x03020100);
+  const __m256i dash = _mm256_set_epi64x(0x0000000000000000ull, 0x2d000000002d0000ull, 0x00002d000000002d, 0x0000000000000000ull);
+
+  __m256i resd = _mm256_shuffle_epi8(res, dash_shuffle);
+  resd = _mm256_or_si256(resd, dash);
+
+  _mm256_storeu_si256((__m256i*)mem, betole256(resd));
+  *(uint16_t*)(mem+16) = betole16(_mm256_extract_epi16(res, 7));
+  *(uint32_t*)(mem+32) = betole32(_mm256_extract_epi32(res, 7));
+}
+
+/*
+  Converts an UUIDv4 string representation to a 128-bits unsigned int.
+  Uses SIMD via Intel's AVX2 instruction set.
+ */
+__m128i inline stom128i(const char* mem) {
+  // Remove dashes and pack hex ascii bytes in a 256-bits int
+  const __m256i dash_shuffle = _mm256_set_epi32(0x80808080, 0x0f0e0d0c, 0x0b0a0908, 0x06050403, 0x80800f0e, 0x0c0b0a09, 0x07060504, 0x03020100);
+
+  __m256i x = betole256(_mm256_loadu_si256((__m256i*)mem));
+  x = _mm256_shuffle_epi8(x, dash_shuffle);
+  x = _mm256_insert_epi16(x, betole16(*(uint16_t*)(mem+16)), 7);
+  x = _mm256_insert_epi32(x, betole32(*(uint32_t*)(mem+32)), 7);
+
+  // Build a mask to apply a different offset to alphas and digits
+  const __m256i sub = _mm256_set1_epi8(0x2F);
+  const __m256i mask = _mm256_set1_epi8(0x20);
+  const __m256i alpha_offset = _mm256_set1_epi8(0x28);
+  const __m256i digits_offset = _mm256_set1_epi8(0x01);
+  const __m256i unweave = _mm256_set_epi32(0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200, 0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200);
+  const __m256i shift = _mm256_set_epi32(0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004);
+
+  // Translate ascii bytes to their value
+  // i.e. 0x3132333435363738 -> 0x0102030405060708
+  // Shift hi-digits
+  // i.e. 0x0102030405060708 -> 0x1002300450067008
+  // Horizontal add
+  // i.e. 0x1002300450067008 -> 0x12345678
+  __m256i a = _mm256_sub_epi8(x, sub);
+  __m256i alpha = _mm256_slli_epi64(_mm256_and_si256(a, mask), 2);
+  __m256i sub_mask = _mm256_blendv_epi8(digits_offset, alpha_offset, alpha);
+  a = _mm256_sub_epi8(a, sub_mask);
+  a = _mm256_shuffle_epi8(a, unweave);
+  a = _mm256_sllv_epi32(a, shift);
+  a = _mm256_hadd_epi32(a, _mm256_setzero_si256());
+  a = _mm256_permute4x64_epi64(a, 0b00001000);
+
+  return _mm256_castsi256_si128(a);
+}
+
+/*
+ * UUIDv4 (random 128-bits) RFC-4122
+ */
+class UUID {
+  public:
+    UUID()
+    {}
+
+    UUID(const UUID &other) {
+      __m128i x = _mm_load_si128((__m128i*)other.data);
+      _mm_store_si128((__m128i*)data, x);
+    }
+
+    /* Builds a 128-bits UUID */
+    UUID(__m128i uuid) {
+      _mm_store_si128((__m128i*)data, uuid);
+    }
+
+    UUID(uint64_t x, uint64_t y) {
+      __m128i z = _mm_set_epi64x(x, y);
+      _mm_store_si128((__m128i*)data, z);
+    }
+
+    UUID(const uint8_t* bytes) {
+      __m128i x = _mm_loadu_si128((__m128i*)bytes);
+      _mm_store_si128((__m128i*)data, x);
+    }
+
+    /* Builds an UUID from a byte string (16 bytes long) */
+    explicit UUID(const std::string &bytes) {
+      __m128i x = betole128(_mm_loadu_si128((__m128i*)bytes.data()));
+      _mm_store_si128((__m128i*)data, x);
+    }
+
+    /* Static factory to parse an UUID from its string representation */
+    static UUID fromStrFactory(const std::string &s) {
+      return fromStrFactory(s.c_str());
+    }
+
+    static UUID fromStrFactory(const char* raw) {
+      return UUID(stom128i(raw));
+    }
+
+    void fromStr(const char* raw) {
+      _mm_store_si128((__m128i*)data, stom128i(raw));
+    }
+
+    UUID& operator=(const UUID &other) {
+      if (&other == this) {
+        return *this;
+      }
+      __m128i x = _mm_load_si128((__m128i*)other.data);
+      _mm_store_si128((__m128i*)data, x);
+      return *this;
+    }
+
+    friend bool operator==(const UUID &lhs, const UUID &rhs) {
+      __m128i x = _mm_load_si128((__m128i*)lhs.data);
+      __m128i y = _mm_load_si128((__m128i*)rhs.data);
+
+      __m128i neq = _mm_xor_si128(x, y);
+      return _mm_test_all_zeros(neq, neq);
+    }
+
+    friend bool operator<(const UUID &lhs, const UUID &rhs) {
+      // There are no trivial 128-bits comparisons in SSE/AVX
+      // It's faster to compare two uint64_t
+      uint64_t *x = (uint64_t*)lhs.data;
+      uint64_t *y = (uint64_t*)rhs.data;
+      return *x < *y || (*x == *y && *(x + 1) < *(y + 1));
+    }
+
+    friend bool operator!=(const UUID &lhs, const UUID &rhs) { return !(lhs == rhs); }
+    friend bool operator> (const UUID &lhs, const UUID &rhs) { return rhs < lhs; }
+    friend bool operator<=(const UUID &lhs, const UUID &rhs) { return !(lhs > rhs); }
+    friend bool operator>=(const UUID &lhs, const UUID &rhs) { return !(lhs < rhs); }
+
+    /* Serializes the uuid to a byte string (16 bytes) */
+    std::string bytes() const {
+      std::string mem;
+      bytes(mem);
+      return mem;
+    }
+
+    void bytes(std::string &out) const {
+      out.resize(sizeof(data));
+      bytes((char*)out.data());
+    }
+
+    void bytes(char* bytes) const {
+      __m128i x = betole128(_mm_load_si128((__m128i*)data));
+      _mm_storeu_si128((__m128i*)bytes, x);
+    }
+
+    /* Converts the uuid to its string representation */
+    std::string str() const {
+      std::string mem;
+      str(mem);
+      return mem;
+    }
+
+    void str(std::string &s) const {
+      s.resize(36);
+      str((char*)s.data());
+    }
+
+    void str(char *res) const {
+      __m128i x = _mm_load_si128((__m128i*)data);
+      m128itos(x, res);
+    }
+
+    friend std::ostream& operator<< (std::ostream& stream, const UUID& uuid) {
+      return stream << uuid.str();
+    }
+
+    friend std::istream& operator>> (std::istream& stream, UUID& uuid) {
+      std::string s;
+      stream >> s;
+      uuid = fromStrFactory(s);
+      return stream;
+    }
+
+    size_t hash() const {
+      const uint64_t a = *((uint64_t*)data);
+      const uint64_t b = *((uint64_t*)&data[8]);
+      return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2));
+    }
+
+  private:
+    alignas(128) uint8_t data[16];
+};
+
+/*
+  Generates UUIDv4 from a provided random generator (c++11 <random> module)
+  std::mt19937_64 is highly recommended as it has a SIMD implementation that
+  makes it very fast and it produces high quality randomness.
+ */
+template <typename RNG>
+class UUIDGenerator {
+  public:
+    UUIDGenerator() : generator(new RNG(std::random_device()())), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
+    {}
+
+    UUIDGenerator(uint64_t seed) : generator(new RNG(seed)), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
+    {}
+
+    UUIDGenerator(RNG &gen) : generator(gen), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max())
+    {}
+
+    /* Generates a new UUID */
+    UUID getUUID() {
+      // The two masks set the uuid version (4) and variant (1)
+      const __m128i and_mask = _mm_set_epi64x(0xFFFFFFFFFFFFFF3Full, 0xFF0FFFFFFFFFFFFFull);
+      const __m128i or_mask =  _mm_set_epi64x(0x0000000000000080ull, 0x0040000000000000ull);
+      __m128i n = _mm_set_epi64x(distribution(*generator), distribution(*generator));
+      __m128i uuid = _mm_or_si128(_mm_and_si128(n, and_mask), or_mask);
+
+      return UUID(uuid);
+    }
+
+  private:
+    std::shared_ptr<RNG> generator;
+    std::uniform_int_distribution<uint64_t> distribution;
+};
+
+}
+
+namespace std {
+  template <> struct hash<UUIDv4::UUID>
+  {
+    size_t operator()(const UUIDv4::UUID & uuid) const
+    {
+      return uuid.hash();
+    }
+  };
+}
--- a/plugin/business/doh/src/logger.cpp
+++ b/plugin/business/doh/src/logger.cpp
@@ -328,25 +328,20 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
    common_obj = cJSON_CreateObject();
    cur_time = time(NULL);

-    cJSON_AddNumberToObject(common_obj, "common_start_time", cur_time);
-    cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time);
+    cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", cur_time);
+    cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", cur_time);
    cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]);
-    cJSON_AddStringToObject(common_obj, "common_schema_type", "DoH");
+    cJSON_AddStringToObject(common_obj, "decoded_as", "DoH");

    char opt_val[24] = { 0 };
-    uint16_t opt_out_size; unsigned int common_direction=0;
+    uint16_t opt_out_size;
    struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream);
    if (cmsg != NULL)
    {
        int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size);
        if (ret == 0)
        {
-			cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val);
-        }
-		ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size);
-        if (ret==0)
-        {
-			cJSON_AddNumberToObject(common_obj, "common_direction", common_direction);	//0：域内->域外，1：域外->域内，描述的是CLIENT_IP信息
+			cJSON_AddStringToObject(common_obj, "session_id", opt_val);
        }
    }

@@ -371,24 +366,22 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
    switch (addr->addrtype)
    {
    case TFE_ADDR_STREAM_TUPLE4_V4:
-        cJSON_AddNumberToObject(common_obj, "common_address_type", 4);
+        cJSON_AddNumberToObject(common_obj, "address_type", 4);
        inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str));
        inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str));
-        cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
-        cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
-        cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source));
-        cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest));
-        cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP");
+        cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
+        cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
+        cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source));
+        cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest));
        break;
    case TFE_ADDR_STREAM_TUPLE4_V6:
-        cJSON_AddNumberToObject(common_obj, "common_address_type", 6);
+        cJSON_AddNumberToObject(common_obj, "address_type", 6);
        inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str));
        inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str));
-        cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
-        cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
-        cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source));
-        cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest));
-        cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP");
+        cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
+        cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
+        cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source));
+        cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest));
        break;
    default:
        break;
@@ -397,20 +390,21 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
    tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num));
    tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num));

-    cJSON_AddNumberToObject(common_obj, "common_link_id", 0);
-    cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double
-    cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str);
-	cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id);
-	cJSON_AddNumberToObject(common_obj, "common_vsys_id", ctx->vsys_id);
-    cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id);
-    cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id);
-    cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num);
-    cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num);
+   	cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
+	cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
+    cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);
+	cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id);
+	cJSON_AddNumberToObject(common_obj, "vsys_id", ctx->vsys_id);
+    cJSON_AddStringToObject(common_obj, "device_id", handle->device_id);
+    cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
+    cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
    cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url);
    cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host);
+	cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
+
 	if(handle->effective_device_tag)
 	{
-		cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag);
+		cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag);
 	}

    for (size_t i = 0; i < sizeof(req_fields) / sizeof(struct json_spec); i++)
@@ -432,19 +426,19 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c

    if (ctx->location_client)
    {
-        cJSON_AddStringToObject(common_obj, "common_client_location", ctx->location_client);
+        cJSON_AddStringToObject(common_obj, "client_geolocation", ctx->location_client);
    }
    if (ctx->location_server)
    {
-        cJSON_AddStringToObject(common_obj, "common_server_location", ctx->location_server);
+        cJSON_AddStringToObject(common_obj, "server_geolocation", ctx->location_server);
    }
    if (ctx->asn_client)
    {
-        cJSON_AddStringToObject(common_obj, "common_client_asn", ctx->asn_client);
+        cJSON_AddStringToObject(common_obj, "client_asn", ctx->asn_client);
    }
    if (ctx->asn_server)
    {
-        cJSON_AddStringToObject(common_obj, "common_server_asn", ctx->asn_server);
+        cJSON_AddStringToObject(common_obj, "server_asn", ctx->asn_server);
    }

    add_dns_info_to_log(common_obj, dns_info);
@@ -462,11 +456,14 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
            continue;
        }

+		cJSON *proxy_rule_list=NULL;
+		int config_id[1]={0};
+
        per_hit_obj = cJSON_Duplicate(common_obj, 1);
-        cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", result[i].config_id);
-        cJSON_AddNumberToObject(per_hit_obj, "common_service", result[i].service_id);
-        cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE);
-        cJSON_AddStringToObject(per_hit_obj, "common_sub_action", "redirect");
+		config_id[0]=result[i].config_id;
+		proxy_rule_list = cJSON_CreateIntArray(config_id, 1);
+		cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list);
+        cJSON_AddStringToObject(per_hit_obj, "proxy_action", "redirect");

        log_payload = cJSON_PrintUnformatted(per_hit_obj);

--- a/plugin/business/tsg-http/src/tsg_logger.cpp
+++ b/plugin/business/tsg-http/src/tsg_logger.cpp
@@ -43,6 +43,18 @@ enum _log_action //Bigger action number is prior.
 	__LG_ACTION_MAX
 };

+#define get_time_ms(tv) ((long long)(tv.tv_sec) * 1000 + (long long)(tv.tv_usec) / 1000)
+
+#include "uuid_v4.h"
+UUIDv4::UUIDGenerator<std::mt19937_64> uuidGenerator;
+
+void get_http_body_uuid(char *uuid)
+{
+    UUIDv4::UUID uid = uuidGenerator.getUUID();
+    uid.str(uuid);
+	return;
+}
+
 struct proxy_logger*  proxy_log_handle_create(const char* profile, const char* section, void* local_logger)
 {
 	struct tango_cache_parameter *log_file_upload_para=NULL;
@@ -115,8 +127,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 	char* log_payload=NULL;
 	int kafka_status=0;
 	int send_cnt=0;
-	int tmp=0;
-	time_t cur_time;
+	struct timeval cur_time;
 	char src_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};
 	char dst_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0};

@@ -145,14 +156,15 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 	}

 	common_obj=cJSON_CreateObject();
-	cur_time = time(NULL);
+	gettimeofday(&cur_time, NULL);

-	cJSON_AddNumberToObject(common_obj, "common_start_time", http->start_time);
-	cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time);
+	cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time));
+	cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
 	cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]);
-	cJSON_AddStringToObject(common_obj, "common_schema_type", "HTTP");
+    cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP");

-	unsigned int common_direction=0, category_id_val[64]={0};
+
+	unsigned int category_id_val[64]={0};
 	char opt_val[24]={0}; uint16_t opt_out_size;
 	struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream);
 	if (cmsg!=NULL)
@@ -160,17 +172,12 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 		int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size);
 		if (ret==0)
 		{
-			cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val);
+			cJSON_AddStringToObject(common_obj, "session_id", opt_val);
 		}
-		ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size);
-        if (ret==0)
-        {
-			cJSON_AddNumberToObject(common_obj, "common_direction", common_direction);	//69：域内->域外，73：域外->域内，描述的是CLIENT_IP信息
-        }
 		ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val));
 		if (ret>0)
 		{
-			cJSON_AddItemToObject(common_obj, "common_service_category", cJSON_CreateIntArray((const int*)category_id_val, ret));
+			cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret));
 		}
 	}

@@ -189,30 +196,29 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 		struct tfe_http_resp_spec resp_spec=http->resp->resp_spec;
 		asprintf(&response_line, "HTTP/%d.%d %d OK", http->major_version, http->minor_version, resp_spec.resp_code);
 		cJSON_AddStringToObject(common_obj, "http_response_line", response_line);
+		cJSON_AddNumberToObject(common_obj, "http_status_code", resp_spec.resp_code);
 		free(response_line);
 	}

 	switch(addr->addrtype)
 	{
 		case TFE_ADDR_STREAM_TUPLE4_V4:
-			cJSON_AddNumberToObject(common_obj, "common_address_type", 4);
+			cJSON_AddNumberToObject(common_obj, "address_type", 4);
 			inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str));
 			inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str));
-			cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
-			cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
-			cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source));
-			cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest));
-			cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP");
+			cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
+			cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
+			cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source));
+			cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest));
 			break;
 		case TFE_ADDR_STREAM_TUPLE4_V6:
-			cJSON_AddNumberToObject(common_obj, "common_address_type", 6);
+			cJSON_AddNumberToObject(common_obj, "address_type", 6);
 			inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str));
 			inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str));
-			cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str);
-			cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str);
-			cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source));
-			cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest));
-			cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP");
+			cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str);
+			cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str);
+			cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source));
+			cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest));
 			break;
 		default:
 			break;
@@ -230,19 +236,20 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 		s2c_byte_num = log_msg->s2c_byte_num;
 	}

-	cJSON_AddNumberToObject(common_obj, "common_link_id", 0);
-	cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3);	//1:c2s, 2:s2c, 3:double
-	cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str);
-	cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id);
-	cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id);
-	cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id);
-	cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num);
-	cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num);
+	cJSON_AddNumberToObject(common_obj, "out_link_id", 0);
+	cJSON_AddNumberToObject(common_obj, "in_link_id", 0);
+	cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str);
+	cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id);
+	cJSON_AddStringToObject(common_obj, "device_id", handle->device_id);
+	cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num);
+	cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num);
 	cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url);
 	cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host);
+	cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host);
+
 	if(handle->effective_device_tag)
 	{
-		cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag);
+		cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag);
 	}

 	for(size_t i=0;i<sizeof(req_fields)/sizeof(struct json_spec);i++)
@@ -262,11 +269,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 		}
 	}

-	char log_file_upload_req_path[TFE_STRING_MAX]={0}, cont_type_whole[TFE_STRING_MAX]={0};
-	char log_file_upload_resp_path[TFE_STRING_MAX]={0};
-	memset(log_file_upload_req_path, 0, sizeof(log_file_upload_req_path));
-	memset(log_file_upload_resp_path, 0, sizeof(log_file_upload_resp_path));
-	memset(cont_type_whole, 0, sizeof(cont_type_whole));
+	#define FILE_CHUNK_UUID_LEN  40
+    char uuid[FILE_CHUNK_UUID_LEN]={0};

 	for(size_t i=0; i<log_msg->result_num; i++)
 	{
@@ -280,81 +284,34 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 			continue;
 		}

-		struct tango_cache_meta_put meta;
-		char* log_file_key=NULL;;
-		const char* cont_type_val;
 		if(log_msg->req_body!=NULL)
 		{
-			if(log_file_upload_req_path[0] != '\0')
+			if(uuid[0] != '\0')
 			{
-				cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path);
+				cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
 			}
 			else
 			{
-				memset(&meta, 0, sizeof(meta));
-				asprintf(&log_file_key, "%s.reqbody", http->req->req_spec.url);
-				meta.url=log_file_key;
-				cont_type_val=tfe_http_std_field_read(http->req, TFE_HTTP_CONT_TYPE);
-				if(cont_type_val!=NULL)
-				{
-					snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
-					meta.std_hdr[0]=cont_type_whole;
-				}
-				meta.user_log_name=1;
-				tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
-																log_msg->req_body,
-																&meta,
-																log_file_upload_req_path, sizeof(log_file_upload_req_path));
-				if(tmp==0)
-				{
-					cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path);
-				}
-				else
-				{
-					TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed.");
-				}
-				free(log_file_key);
+				get_http_body_uuid(uuid);
+				cJSON_AddStringToObject(common_obj, "http_request_body", uuid);
 			}
 		}
 		if(log_msg->resp_body!=NULL)
 		{
-			if(log_file_upload_resp_path[0] != '\0')
+			if(uuid[0] != '\0')
 			{
-				cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path);
+				cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
 			}
 			else
 			{
-				memset(&meta, 0, sizeof(meta));
-				asprintf(&log_file_key, "%s.respbody", http->req->req_spec.url);
-				meta.url=log_file_key;
-				cont_type_val=tfe_http_std_field_read(http->resp, TFE_HTTP_CONT_TYPE);
-				if(cont_type_val!=NULL)
-				{
-					snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val);
-					meta.std_hdr[0]=cont_type_whole;
-				}
-				meta.user_log_name=1;
-				tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL,
-																log_msg->resp_body,
-																&meta,
-																log_file_upload_resp_path, sizeof(log_file_upload_resp_path));
-
-				if(tmp==0)
-				{
-					cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path);
-				}
-				else
-				{
-					TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed.");
-				}
-				free(log_file_key);
+				get_http_body_uuid(uuid);
+				cJSON_AddStringToObject(common_obj, "http_response_body", uuid);
 			}
 		}
 	}

 	for(size_t i=0; i<log_msg->result_num; i++)
 	{
-
 		TFE_LOG_DEBUG(handle->local_logger, "URL: %s, policy_id: %lld, service: %d, do_log:%d",
 						http->req->req_spec.url,
 						log_msg->result[i].config_id,
@@ -366,27 +323,38 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg)
 			continue;
 		}

+		cJSON *proxy_rule_list=NULL;
+		int config_id[1]={0};
+
 		per_hit_obj=cJSON_Duplicate(common_obj, 1);
-		cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", log_msg->result[i].config_id);
-		cJSON_AddNumberToObject(per_hit_obj, "common_service", log_msg->result[i].service_id);
-		cJSON_AddNumberToObject(per_hit_obj, "common_vsys_id", log_msg->result[i].vsys_id);
-		cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE);
+		config_id[0]=log_msg->result[i].config_id;
+		proxy_rule_list = cJSON_CreateIntArray(config_id, 1);
+		cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list);
+		cJSON_AddNumberToObject(per_hit_obj, "vsys_id", log_msg->result[i].vsys_id);
 		if(log_msg->result[i].action == LG_ACTION_MANIPULATE)
 		{
-			cJSON_AddStringToObject(per_hit_obj, "common_sub_action", manipulate_action_map[log_msg->action]);
+			cJSON_AddStringToObject(per_hit_obj, "proxy_action", manipulate_action_map[log_msg->action]);
 			cJSON_AddNumberToObject(per_hit_obj, "http_action_file_size", log_msg->inject_sz);
 		}
 		else
 		{
-			cJSON_AddStringToObject(per_hit_obj, "common_sub_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
+			cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]);
 		}
 		if(log_msg->location_client)
 		{
-			cJSON_AddStringToObject(per_hit_obj, "common_client_location", log_msg->location_client);
+			cJSON_AddStringToObject(per_hit_obj, "client_geolocation", log_msg->location_client);
 		}
 		if(log_msg->location_server)
 		{
-			cJSON_AddStringToObject(per_hit_obj, "common_server_location", log_msg->location_server);
+			cJSON_AddStringToObject(per_hit_obj, "server_geolocation", log_msg->location_server);
+		}
+		if(log_msg->asn_client)
+    	{
+			cJSON_AddStringToObject(common_obj, "client_asn", log_msg->asn_client);
+		}
+		if (log_msg->asn_server)
+		{
+		    cJSON_AddStringToObject(common_obj, "server_asn", log_msg->asn_server);
 		}

 		log_payload = cJSON_PrintUnformatted(per_hit_obj);
--- a/plugin/protocol/http/src/http_half.cpp
+++ b/plugin/protocol/http/src/http_half.cpp
@@ -1136,7 +1136,7 @@ struct http_session_private * hs_private_create(struct http_connection_private *
 	__hs_private->hs_public.req = hf_private_req != NULL ? to_hf_public(hf_private_req) : NULL;
 	__hs_private->hs_public.resp = hf_private_req != NULL ? to_hf_public(hf_private_resp) : NULL;
 	__hs_private->hs_public.session_id = hc_private->session_id_counter++;
-	__hs_private->hs_public.start_time=time(NULL);
+	gettimeofday(&(__hs_private->hs_public.start_time), NULL);

 	/* HS-PRIVATE*/
 	__hs_private->hc_private = hc_private;
--- a/plugin/protocol/http2/src/http2_stream.cpp
+++ b/plugin/protocol/http2/src/http2_stream.cpp
@@ -1270,7 +1270,7 @@ static int tfe_half_session_init(struct tfe_h2_session *h2_session, int32_t stre
 		struct tfe_h2_half_private *req = h2_session->req;
 		tfe_session->ops = &http2_session_ops;
 		tfe_session->req = &req->half_public;
-		tfe_session->start_time=time(NULL);
+		gettimeofday(&tfe_session->start_time, NULL);
 		tfe_session->session_id = stream_id;

 	}