TSG-2967 TFE 使用 JA3 优化 SSL pinning

common/include/tfe_cmsg.h

@@ -75,6 +75,9 @@ enum tfe_cmsg_tlv_type
     TFE_CMSG_SRC_IP_LOCATION_CITY,    // string max size 256
     TFE_CMSG_DST_IP_LOCATION_CITY,    // string max size 256
 
+    /* SSL ja3 fingerprint */
+    TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, // string max size 32
+
     /* Add new cmsg here */
     /* Add new cmsg here */
     /* Add new cmsg here */

conf/tfe/tfe.conf

@@ -35,6 +35,7 @@ watchdog_switch=1
 watchdog_port=2476
 
 [ssl]
+ssl_ja3_debug=0
 ssl_max_version=tls13
 ssl_min_version=ssl3
 ssl_compression=1

platform/CMakeLists.txt

@@ -3,7 +3,7 @@ find_package(SYSTEMD REQUIRED)
 add_executable(tfe src/acceptor_kni_v1.cpp src/acceptor_kni_v2.cpp src/ssl_stream.cpp src/key_keeper.cpp src/ssl_fetch_cert.cpp
 	src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp
 	src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp
-	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp)
+	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/ssl_ja3.cpp)
 
 target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)
 target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)

platform/include/internal/ssl_ja3.h

@@ -0,0 +1,10 @@
+#pragma once
+
+
+struct ssl_ja3 {
+    char *ja3;
+    char *ja3_hash;
+};
+
+struct ssl_ja3 *ssl_ja3_generate_fingerprint(const unsigned char *data, int len);
+void ssl_ja3_free(struct ssl_ja3 *fingerprint);

platform/include/internal/ssl_service_cache.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <tfe_stream.h>
 #include <tfe_types.h>
 #include <ssl_utils.h>
 #include <event2/event.h>
@@ -42,6 +43,6 @@ struct ssl_service_cache
 struct ssl_service_cache* ssl_service_cache_create(unsigned int slot_size, unsigned int expire_seconds, int fail_as_pinning_cnt, int fail_as_proto_err_cnt, int succ_as_app_not_pinning_cnt, int fail_time_win);
 void ssl_service_cache_destroy(struct ssl_service_cache* cache);
 
-int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, struct ssl_service_status* result);
+int ssl_service_cache_read(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, struct ssl_service_status *result);
-void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, const struct ssl_service_status* status);
+void ssl_service_cache_write(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, const struct ssl_service_status *status);
-void ssl_service_cache_stat(struct ssl_service_cache* svc_cache, struct ssl_service_cache_statistics* result);
+void ssl_service_cache_stat(struct ssl_service_cache *svc_cache, struct ssl_service_cache_statistics *result);

platform/src/ssl_ja3.cpp

@@ -0,0 +1,603 @@
+#include <openssl/md5.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ssl_ja3.h>
+
+#ifndef MIN
+#define MIN(a, b) ((a) > (b) ? (b) : (a))
+#endif
+
+#define BSB_INIT(b, buffer, size)                     \
+    do                                                \
+    {                                                 \
+        (b).buf = (unsigned char *)buffer;            \
+        (b).ptr = (unsigned char *)buffer;            \
+        int s = (int)size;                            \
+        if ((buffer == NULL) || (s < 0))              \
+            (b).end = 0;                              \
+        else                                          \
+            (b).end = (unsigned char *)buffer + size; \
+    } while (0)
+
+#define BSB_SET_ERROR(b) ((b).end = NULL)
+#define BSB_IS_ERROR(b) ((b).end == NULL)
+#define BSB_NOT_ERROR(b) ((b).end != NULL)
+#define BSB_LENGTH(b) ((b).ptr - (b).buf)
+#define BSB_POSITION BSB_LENGTH
+#define BSB_SIZE(b) ((b).end - (b).buf)
+#define BSB_REMAINING(b) ((b).end ? (b).end - (b).ptr : 0)
+#define BSB_WORK_PTR(b) ((b).ptr)
+
+#define BSB_EXPORT_u08(b, x)                   \
+    do                                         \
+    {                                          \
+        if ((b).ptr && (b).ptr + 1 <= (b).end) \
+        {                                      \
+            *(((b).ptr)++) = (unsigned char)x; \
+        }                                      \
+        else                                   \
+            BSB_SET_ERROR(b);                  \
+    } while (0)
+
+#define BSB_EXPORT_u16(b, x)                    \
+    do                                          \
+    {                                           \
+        if ((b).ptr && (b).ptr + 2 <= (b).end)  \
+        {                                       \
+            uint16_t t = (uint16_t)x;           \
+            *(((b).ptr)++) = (t & 0xff00) >> 8; \
+            *(((b).ptr)++) = (t & 0x00ff);      \
+        }                                       \
+        else                                    \
+            BSB_SET_ERROR(b);                   \
+    } while (0)
+
+#define BSB_EXPORT_u32(b, x)                         \
+    do                                               \
+    {                                                \
+        if ((b).ptr && (b).ptr + 4 <= (b).end)       \
+        {                                            \
+            uint32_t t = x;                          \
+            *(((b).ptr)++) = (t & 0xff000000) >> 24; \
+            *(((b).ptr)++) = (t & 0x00ff0000) >> 16; \
+            *(((b).ptr)++) = (t & 0x0000ff00) >> 8;  \
+            *(((b).ptr)++) = (t & 0x000000ff);       \
+        }                                            \
+        else                                         \
+            BSB_SET_ERROR(b);                        \
+    } while (0)
+
+#define BSB_EXPORT_ptr(b, x, size)       \
+    do                                   \
+    {                                    \
+        if ((x || size == 0) &&          \
+            (b).ptr + size <= (b).end && \
+            (b).ptr + size >= (b).buf)   \
+        {                                \
+            memcpy((b).ptr, x, size);    \
+            (b).ptr += size;             \
+        }                                \
+        else                             \
+            BSB_SET_ERROR(b);            \
+    } while (0)
+
+#define BSB_EXPORT_ptr_some(b, x, size)           \
+    do                                            \
+    {                                             \
+        if ((b).ptr + size <= (b).end)            \
+        {                                         \
+            memcpy((b).ptr, x, size);             \
+            (b).ptr += size;                      \
+        }                                         \
+        else if (BSB_NOT_ERROR(b))                \
+        {                                         \
+            memcpy((b).ptr, x, BSB_REMAINING(b)); \
+            (b).ptr += BSB_REMAINING(b);          \
+        }                                         \
+    } while (0)
+
+#define BSB_EXPORT_cstr(b, x)          \
+    do                                 \
+    {                                  \
+        const int size = sizeof x - 1; \
+        if ((b).ptr + size <= (b).end) \
+        {                              \
+            memcpy((b).ptr, x, size);  \
+            (b).ptr += size;           \
+        }                              \
+        else                           \
+            BSB_SET_ERROR(b);          \
+    } while (0)
+
+#define BSB_EXPORT_skip(b, size)         \
+    do                                   \
+    {                                    \
+        if ((b).ptr + size <= (b).end && \
+            (b).ptr + size >= (b).buf)   \
+        {                                \
+            (b).ptr += size;             \
+            if ((b).ptr < (b).buf)       \
+                (b).end = 0;             \
+        }                                \
+        else                             \
+            BSB_SET_ERROR(b);            \
+    } while (0)
+
+#define BSB_EXPORT_rewind(b, size)       \
+    do                                   \
+    {                                    \
+        if ((b).ptr - size <= (b).end && \
+            (b).ptr - size >= (b).buf)   \
+        {                                \
+            (b).ptr -= size;             \
+            if ((b).ptr < (b).buf)       \
+                (b).end = 0;             \
+        }                                \
+        else                             \
+        {                                \
+            BSB_SET_ERROR(b);            \
+        }                                \
+    } while (0)
+
+#if defined(C9X)
+
+#define BSB_EXPORT_sprintf(b, ...)              \
+    do                                          \
+    {                                           \
+        if ((b).end != 0)                       \
+        {                                       \
+            int l = snprintf((char *)(b).ptr,   \
+                             (b).end - (b).ptr, \
+                             __VA_ARGS__);      \
+            if (l <= (b).end - (b).ptr)         \
+            {                                   \
+                (b).ptr += l;                   \
+            }                                   \
+            else                                \
+            {                                   \
+                BSB_SET_ERROR(b);               \
+            }                                   \
+        }                                       \
+    } while (0)
+
+#else
+
+#define BSB_EXPORT_sprintf(b, args...)          \
+    do                                          \
+    {                                           \
+        if ((b).end != 0)                       \
+        {                                       \
+            int l = snprintf((char *)(b).ptr,   \
+                             (b).end - (b).ptr, \
+                             ##args);           \
+            if (l <= (b).end - (b).ptr)         \
+            {                                   \
+                (b).ptr += l;                   \
+            }                                   \
+            else                                \
+            {                                   \
+                BSB_SET_ERROR(b);               \
+            }                                   \
+        }                                       \
+    } while (0)
+#endif
+
+#define BSB_IMPORT_u08(b, x)                   \
+    do                                         \
+    {                                          \
+        if ((b).ptr && (b).ptr + 1 <= (b).end) \
+        {                                      \
+            x = *(((b).ptr)++);                \
+        }                                      \
+        else                                   \
+            BSB_SET_ERROR(b);                  \
+    } while (0)
+
+#define BSB_IMPORT_u16(b, x)                    \
+    do                                          \
+    {                                           \
+        if ((b).ptr && (b).ptr + 2 <= (b).end)  \
+        {                                       \
+            x = ((uint16_t)((b).ptr)[0]) << 8 | \
+                ((uint16_t)((b).ptr)[1]);       \
+            (b).ptr += 2;                       \
+        }                                       \
+        else                                    \
+            BSB_SET_ERROR(b);                   \
+    } while (0)
+
+#define BSB_IMPORT_u24(b, x)                     \
+    do                                           \
+    {                                            \
+        if ((b).ptr && (b).ptr + 3 <= (b).end)   \
+        {                                        \
+            x = ((uint32_t)((b).ptr)[0]) << 16 | \
+                ((uint32_t)((b).ptr)[1]) << 8 |  \
+                ((uint32_t)((b).ptr)[2]);        \
+            (b).ptr += 3;                        \
+        }                                        \
+        else                                     \
+            BSB_SET_ERROR(b);                    \
+    } while (0)
+
+#define BSB_IMPORT_u32(b, x)                     \
+    do                                           \
+    {                                            \
+        if ((b).ptr && (b).ptr + 4 <= (b).end)   \
+        {                                        \
+            x = ((uint32_t)((b).ptr)[0]) << 24 | \
+                ((uint32_t)((b).ptr)[1]) << 16 | \
+                ((uint32_t)((b).ptr)[2]) << 8 |  \
+                ((uint32_t)((b).ptr)[3]);        \
+            (b).ptr += 4;                        \
+        }                                        \
+        else                                     \
+            BSB_SET_ERROR(b);                    \
+    } while (0)
+
+#define BSB_LEXPORT_u08(b, x) BSB_EXPORT_u08(b, x)
+
+#define BSB_LEXPORT_u16(b, x)                   \
+    do                                          \
+    {                                           \
+        if ((b).ptr + 2 <= (b).end)             \
+        {                                       \
+            uint16_t t = (uint16_t)x;           \
+            *(((b).ptr)++) = (t & 0x00ff);      \
+            *(((b).ptr)++) = (t & 0xff00) >> 8; \
+        }                                       \
+        else                                    \
+            BSB_SET_ERROR(b);                   \
+    } while (0)
+
+#define BSB_LEXPORT_u32(b, x)                        \
+    do                                               \
+    {                                                \
+        if ((b).ptr + 4 <= (b).end)                  \
+        {                                            \
+            uint32_t t = x;                          \
+            *(((b).ptr)++) = (t & 0x000000ff);       \
+            *(((b).ptr)++) = (t & 0x0000ff00) >> 8;  \
+            *(((b).ptr)++) = (t & 0x00ff0000) >> 16; \
+            *(((b).ptr)++) = (t & 0xff000000) >> 24; \
+        }                                            \
+        else                                         \
+            BSB_SET_ERROR(b);                        \
+    } while (0)
+
+#define BSB_LIMPORT_u08(b, x) BSB_IMPORT_u08(b, x)
+
+#define BSB_LIMPORT_u16(b, x)                   \
+    do                                          \
+    {                                           \
+        if ((b).ptr + 2 <= (b).end)             \
+        {                                       \
+            x = ((uint16_t)((b).ptr)[1]) << 8 | \
+                ((uint16_t)((b).ptr)[0]);       \
+            (b).ptr += 2;                       \
+        }                                       \
+        else                                    \
+            BSB_SET_ERROR(b);                   \
+    } while (0)
+
+#define BSB_LIMPORT_u24(b, x)                    \
+    do                                           \
+    {                                            \
+        if ((b).ptr + 3 <= (b).end)              \
+        {                                        \
+            x = ((uint32_t)((b).ptr)[2]) << 16 | \
+                ((uint32_t)((b).ptr)[1]) << 8 |  \
+                ((uint32_t)((b).ptr)[0]);        \
+            (b).ptr += 3;                        \
+        }                                        \
+        else                                     \
+            BSB_SET_ERROR(b);                    \
+    } while (0)
+
+#define BSB_LIMPORT_u32(b, x)                    \
+    do                                           \
+    {                                            \
+        if ((b).ptr + 4 <= (b).end)              \
+        {                                        \
+            x = ((uint32_t)((b).ptr)[3]) << 24 | \
+                ((uint32_t)((b).ptr)[2]) << 16 | \
+                ((uint32_t)((b).ptr)[1]) << 8 |  \
+                ((uint32_t)((b).ptr)[0]);        \
+            (b).ptr += 4;                        \
+        }                                        \
+        else                                     \
+            BSB_SET_ERROR(b);                    \
+    } while (0)
+
+#define BSB_IMPORT_ptr(b, x, size)       \
+    do                                   \
+    {                                    \
+        if ((b).ptr + size <= (b).end && \
+            (b).ptr + size >= (b).buf)   \
+        {                                \
+            (x) = (b).ptr;               \
+            (b).ptr += size;             \
+        }                                \
+        else                             \
+        {                                \
+            BSB_SET_ERROR(b);            \
+            x = 0;                       \
+        }                                \
+    } while (0)
+
+#define BSB_LIMPORT_ptr BSB_IMPORT_ptr
+#define BSB_IMPORT_skip BSB_EXPORT_skip
+#define BSB_LIMPORT_skip BSB_EXPORT_skip
+#define BSB_IMPORT_rewind BSB_EXPORT_rewind
+#define BSB_LIMPORT_rewind BSB_EXPORT_rewind
+
+#define BSB_memchr(b, ch, pos)                                 \
+    do                                                         \
+    {                                                          \
+        if (BSB_IS_ERROR(b))                                   \
+        {                                                      \
+            pos = 0;                                           \
+            break;                                             \
+        }                                                      \
+        char *s = memchr((char *)b.ptr, ch, BSB_REMAINING(b)); \
+        if (s)                                                 \
+            pos = (char *)s - (char *)b.ptr;                   \
+        else                                                   \
+            pos = 0;                                           \
+    } while (0)
+
+#define BSB_memcmp(str, b, len) ((b).ptr + len <= (b).end ? memcmp(str, b.ptr, len) : -1)
+
+#define BSB_PEEK(b) ((b).ptr + 1 <= (b).end ? *b.ptr : -1)
+
+#define BSB_IMPORT_zbyte(b, x, size)   \
+    do                                 \
+    {                                  \
+        if ((b).ptr + size <= (b).end) \
+        {                              \
+            memcpy(x, b, size);        \
+            (x)[size] = 0;             \
+            (b).ptr += size;           \
+        }                              \
+        else                           \
+        {                              \
+            BSB_SET_ERROR(b);          \
+            (x)[0] = 0;                \
+        }                              \
+    } while (0)
+
+/* Private data structure */
+typedef struct bsb
+{
+    unsigned char *buf;
+    unsigned char *ptr;
+    unsigned char *end;
+} BSB;
+
+// https://tools.ietf.org/html/draft-davidben-tls-grease-00
+static int tls_is_grease_value(uint32_t val)
+{
+    if ((val & 0x0f) != 0x0a)
+        return 0;
+
+    if ((val & 0xff) != ((val >> 8) & 0xff))
+        return 0;
+
+    return 1;
+}
+
+static int md5sum(const char *str, int len, char *buf, int size)
+{
+    int n;
+    int ret = 0;
+    MD5_CTX ctx;
+    unsigned char tmp[MD5_DIGEST_LENGTH];
+
+    MD5_Init(&ctx);
+    MD5_Update(&ctx, str, len);
+    MD5_Final(tmp, &ctx);
+
+    for (n = 0; n < MD5_DIGEST_LENGTH; n++)
+    {
+        ret += snprintf(buf + ret, size - ret, "%.2x", tmp[n]);
+    }
+
+    return ret;
+}
+
+void ssl_ja3_free(struct ssl_ja3 *fingerprint)
+{
+    if (fingerprint)
+    {
+        if (fingerprint->ja3)
+        {
+            free(fingerprint->ja3);
+            fingerprint->ja3 = NULL;
+        }
+        if (fingerprint->ja3_hash)
+        {
+            free(fingerprint->ja3_hash);
+            fingerprint->ja3_hash = NULL;
+        }
+        free(fingerprint);
+        fingerprint = NULL;
+    }
+}
+
+ /******************************************************************************
+  * JA3 generation algorithm refers to tls_process_client() in
+  * https://github.com/aol/moloch.
+  ******************************************************************************/
+struct ssl_ja3 *ssl_ja3_generate_fingerprint(const unsigned char *data, int len)
+{
+    BSB sslbsb;
+    char ja3[30000];
+    BSB ja3bsb;
+    char ecfja3[1000];
+    BSB ecfja3bsb;
+    char eja3[10000];
+    BSB eja3bsb;
+    char ecja3[10000];
+    BSB ecja3bsb;
+
+    BSB_INIT(sslbsb, data, len);
+    BSB_INIT(ja3bsb, ja3, sizeof(ja3));
+    BSB_INIT(ecja3bsb, ecja3, sizeof(ecja3));
+    BSB_INIT(ecfja3bsb, ecfja3, sizeof(ecfja3));
+    BSB_INIT(eja3bsb, eja3, sizeof(eja3));
+
+    if (BSB_REMAINING(sslbsb) > 5)
+    {
+        unsigned char *ssldata = BSB_WORK_PTR(sslbsb);
+        int ssllen = MIN(BSB_REMAINING(sslbsb) - 5, ssldata[3] << 8 | ssldata[4]);
+
+        BSB pbsb;
+        BSB_INIT(pbsb, ssldata + 5, ssllen);
+
+        if (BSB_REMAINING(pbsb) > 7)
+        {
+            unsigned char *pdata = BSB_WORK_PTR(pbsb);
+            int plen = MIN(BSB_REMAINING(pbsb) - 4, pdata[2] << 8 | pdata[3]);
+
+            uint16_t ver = 0;
+            BSB_IMPORT_skip(pbsb, 4); // type + len
+            BSB_IMPORT_u16(pbsb, ver);
+
+            BSB_EXPORT_sprintf(ja3bsb, "%d,", ver);
+
+            BSB cbsb;
+            BSB_INIT(cbsb, pdata + 6, plen - 2); // The - 4 for plen is done above, confusing
+
+            if (BSB_REMAINING(cbsb) > 32)
+            {
+                BSB_IMPORT_skip(cbsb, 32); // Random
+
+                int skiplen = 0;
+                BSB_IMPORT_u08(cbsb, skiplen); // Session Id Length
+                if (skiplen > 0 && BSB_REMAINING(cbsb) > skiplen)
+                {
+                    // Remove unnecessary code
+                }
+                BSB_IMPORT_skip(cbsb, skiplen); // Session Id
+
+                BSB_IMPORT_u16(cbsb, skiplen); // Ciper Suites Length
+                while (BSB_NOT_ERROR(cbsb) && skiplen > 0)
+                {
+                    uint16_t c = 0;
+                    BSB_IMPORT_u16(cbsb, c);
+                    if (!tls_is_grease_value(c))
+                    {
+                        BSB_EXPORT_sprintf(ja3bsb, "%d-", c);
+                    }
+                    skiplen -= 2;
+                }
+                BSB_EXPORT_rewind(ja3bsb, 1); // Remove last -
+                BSB_EXPORT_u08(ja3bsb, ',');
+
+                BSB_IMPORT_u08(cbsb, skiplen);  // Compression Length
+                BSB_IMPORT_skip(cbsb, skiplen); // Compressions
+
+                if (BSB_REMAINING(cbsb) > 6)
+                {
+                    int etotlen = 0;
+                    BSB_IMPORT_u16(cbsb, etotlen); // Extensions Length
+
+                    etotlen = MIN(etotlen, BSB_REMAINING(cbsb));
+
+                    BSB ebsb;
+                    BSB_INIT(ebsb, BSB_WORK_PTR(cbsb), etotlen);
+
+                    while (BSB_REMAINING(ebsb) > 4)
+                    {
+                        uint16_t etype = 0, elen = 0;
+
+                        BSB_IMPORT_u16(ebsb, etype);
+                        BSB_IMPORT_u16(ebsb, elen);
+
+                        if (!tls_is_grease_value(etype))
+                            BSB_EXPORT_sprintf(eja3bsb, "%d-", etype);
+
+                        if (elen > BSB_REMAINING(ebsb))
+                            break;
+
+                        if (etype == 0)
+                        { // SNI
+                            BSB snibsb;
+                            BSB_INIT(snibsb, BSB_WORK_PTR(ebsb), elen);
+                            BSB_IMPORT_skip(ebsb, elen);
+
+                            int sni = 0;
+                            BSB_IMPORT_u16(snibsb, sni); // list len
+                            if (sni != BSB_REMAINING(snibsb))
+                                continue;
+
+                            BSB_IMPORT_u08(snibsb, sni); // type
+                            if (sni != 0)
+                                continue;
+
+                            BSB_IMPORT_u16(snibsb, sni); // len
+                            if (sni != BSB_REMAINING(snibsb))
+                                continue;
+                        }
+                        else if (etype == 0x000a)
+                        { // Elliptic Curves
+                            BSB bsb;
+                            BSB_INIT(bsb, BSB_WORK_PTR(ebsb), elen);
+                            BSB_IMPORT_skip(ebsb, elen);
+
+                            uint16_t llen = 0;
+                            BSB_IMPORT_u16(bsb, llen); // list len
+                            while (llen > 0 && !BSB_IS_ERROR(bsb))
+                            {
+                                uint16_t c = 0;
+                                BSB_IMPORT_u16(bsb, c);
+                                if (!tls_is_grease_value(c))
+                                {
+                                    BSB_EXPORT_sprintf(ecja3bsb, "%d-", c);
+                                }
+                                llen -= 2;
+                            }
+                            BSB_EXPORT_rewind(ecja3bsb, 1); // Remove last -
+                        }
+                        else if (etype == 0x000b)
+                        { // Elliptic Curves point formats
+                            BSB bsb;
+                            BSB_INIT(bsb, BSB_WORK_PTR(ebsb), elen);
+                            BSB_IMPORT_skip(ebsb, elen);
+
+                            uint16_t llen = 0;
+                            BSB_IMPORT_u08(bsb, llen); // list len
+                            while (llen > 0 && !BSB_IS_ERROR(bsb))
+                            {
+                                uint8_t c = 0;
+                                BSB_IMPORT_u08(bsb, c);
+                                BSB_EXPORT_sprintf(ecfja3bsb, "%d-", c);
+                                llen -= 1;
+                            }
+                            BSB_EXPORT_rewind(ecfja3bsb, 1); // Remove last -
+                        }
+                        else
+                        {
+                            BSB_IMPORT_skip(ebsb, elen);
+                        }
+                    }
+                    BSB_EXPORT_rewind(eja3bsb, 1); // Remove last -
+                }
+            }
+        }
+        BSB_IMPORT_skip(sslbsb, ssllen + 5);
+
+        if (BSB_LENGTH(ja3bsb) > 0 && BSB_NOT_ERROR(ja3bsb) && BSB_NOT_ERROR(ecja3bsb) && BSB_NOT_ERROR(eja3bsb) && BSB_NOT_ERROR(ecfja3bsb))
+        {
+            BSB_EXPORT_sprintf(ja3bsb, "%.*s,%.*s,%.*s", (int)BSB_LENGTH(eja3bsb), eja3, (int)BSB_LENGTH(ecja3bsb), ecja3, (int)BSB_LENGTH(ecfja3bsb), ecfja3);
+            char md5[64] = {0};
+            int len = md5sum(ja3, BSB_LENGTH(ja3bsb), md5, sizeof(md5));
+            struct ssl_ja3 *result = (struct ssl_ja3 *)calloc(sizeof(struct ssl_ja3), 1);
+            asprintf(&(result->ja3), "%.*s", (int)BSB_LENGTH(ja3bsb), ja3);
+            asprintf(&(result->ja3_hash), "%.*s", len, md5);
+            return result;
+        }
+    }
+
+    return NULL;
+}

platform/src/ssl_service_cache.cpp

@@ -86,13 +86,37 @@ static size_t ssl_svc_server_st_mk_key(const struct ssl_chello* chello,  const s
 	free(addr_str);
 	return key_len;
 }
-static size_t ssl_svc_app_st_mk_key(const struct ssl_chello* chello,  const struct tfe_stream_addr * addr, char* key_buff, size_t sz)
+static size_t ssl_svc_app_st_mk_key(const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, char *key_buff, size_t sz)
 {
-	size_t key_len=0;
+    char ja3_val[64] = {0};
-	const char* sip=NULL, *sport=NULL, *dip=NULL, *dport=NULL;
+    uint16_t ja3_len = 0;
-	char * addr_str= tfe_stream_addr_to_str(addr);
+    size_t key_len = 0;
+    const char *sip = NULL, *sport = NULL, *dip = NULL, *dport = NULL;
+    char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr);
+    struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(tcp_stream);
+    if (cmsg != NULL)
+    {
+        int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, (unsigned char *)ja3_val, sizeof(ja3_val), &ja3_len);
+        if (ret != 0)
+        {
+            TFE_LOG_ERROR(g_default_logger, "failed at fetch ssl client ja3 fingerprint from cmsg: %s, %s", strerror(-ret), addr_str);
+        }
+		else
+		{
+            TFE_LOG_DEBUG(g_default_logger, "fetch ssl client ja3 fingerprint:%s addr: %s", ja3_val, addr_str);
+        }	
+    }
     tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport);
 
+    // If ja3 is successfully obtained, use ja3 to generate hashkey
+    if (strlen(ja3_val))
+    {
+        key_len = snprintf(key_buff, sz, "%s:%s", ja3_val, chello->sni ? chello->sni : dip);
+        free(addr_str);
+        return key_len;
+    }
+
+    // otherwise, splicing ssl attributes to generate hashkey
     key_len=snprintf(key_buff, sz, "%d:%d:%s:%s", chello->min_version.ossl_format, 
 												chello->max_version.ossl_format,
 												chello->sni?chello->sni: dip ,
@@ -115,17 +139,17 @@ static size_t ssl_svc_app_st_mk_key(const struct ssl_chello* chello,  const stru
 	free(addr_str);
 	return key_len;
 }
-static size_t ssl_svc_client_st_mk_key(const struct ssl_chello* chello, const struct tfe_stream_addr * addr, char* key_buff, size_t sz)
+static size_t ssl_svc_client_st_mk_key(const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, char *key_buff, size_t sz)
 {
 	size_t key_len=0;
 	const char* sip=NULL, *sport=NULL, *dip=NULL, *dport=NULL;
-	char * addr_str= tfe_stream_addr_to_str(addr);
+    char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr);
     tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport);
 	char chello_id_buff[sz];
 	size_t chello_id_len=0;
 
 	key_len=snprintf(key_buff, sz, "%s:", sip);
-	chello_id_len=ssl_svc_app_st_mk_key(chello, addr, chello_id_buff, sizeof(chello_id_buff));
+    chello_id_len = ssl_svc_app_st_mk_key(chello, tcp_stream, chello_id_buff, sizeof(chello_id_buff));
     memcpy(key_buff+key_len, chello_id_buff, MIN(chello_id_len, sz-key_len));
 	key_len += MIN(chello_id_len, sz-key_len);
 
@@ -305,7 +329,7 @@ static long app_st_write_cb(void * data, const uchar * key, uint size, void * us
 	return 1;
 }
 
-int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, struct ssl_service_status* result)
+int ssl_service_cache_read(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, struct ssl_service_status *result)
 {
 	long cli_st_cb_ret=0, svr_st_cb_ret=0, app_st_cb_ret=0;
 	char temp_key[2048];
@@ -317,24 +341,27 @@ int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl
 		return 0;
 	}
 
-	char * addr_str= tfe_stream_addr_to_str(addr);
+	char * addr_str= tfe_stream_addr_to_str(tcp_stream->addr);
 	memset(hash_key, 0, sizeof(hash_key));
-	temp_key_sz = ssl_svc_client_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+    temp_key_sz = ssl_svc_client_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key));
     hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
 	MESA_htable_search_cb(svc_cache->cli_st_hash, hash_key, (unsigned int) hash_key_sz, cli_st_read_cb, result, &cli_st_cb_ret);
-	TFE_LOG_DEBUG(g_default_logger, "ssl svc read client table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, cli_st_cb_ret, chello->sni, addr_str);
+	TFE_LOG_DEBUG(g_default_logger, "client table, hash:%s, found:%d, sni:%s, addr:%s, mutual:%d, pinning:%d, err:%d",
+				hash_key, cli_st_cb_ret, chello->sni, addr_str, result->is_mutual_auth, result->cli_pinning_status, result->has_protocol_errors);
 
 	memset(hash_key, 0, sizeof(hash_key));
-	temp_key_sz = ssl_svc_server_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+    temp_key_sz = ssl_svc_server_st_mk_key(chello, tcp_stream->addr, temp_key, sizeof(temp_key));
     hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
 	MESA_htable_search_cb(svc_cache->srv_st_hash, hash_key, (unsigned int) hash_key_sz, srv_st_read_cb, result, &svr_st_cb_ret);
-	TFE_LOG_DEBUG(g_default_logger, "ssl svc read server table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, svr_st_cb_ret, chello->sni, addr_str);
+	TFE_LOG_DEBUG(g_default_logger, "server table, hash:%s, found:%d, sni:%s, addr:%s, ct:%d, ev:%d",
+				hash_key, svr_st_cb_ret, chello->sni, addr_str, result->is_ct, result->is_ev);
 
 	memset(hash_key, 0, sizeof(hash_key));
-	temp_key_sz = ssl_svc_app_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+    temp_key_sz = ssl_svc_app_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key));
     hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
 	MESA_htable_search_cb(svc_cache->app_st_hash, hash_key, (unsigned int) hash_key_sz, app_st_read_cb, result, &app_st_cb_ret);
-	TFE_LOG_DEBUG(g_default_logger, "ssl svc read app table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, app_st_cb_ret, chello->sni, addr_str);
+    TFE_LOG_DEBUG(g_default_logger, "app table, hash:%s, found:%d, sni:%s, addr:%s, app_not_pinning:%d",
+				hash_key, app_st_cb_ret, chello->sni, addr_str, result->is_app_not_pinning);
 
     free(addr_str);
 
@@ -348,7 +375,7 @@ int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl
 	}
 }
 
-void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, const struct ssl_service_status* status)
+void ssl_service_cache_write(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, const struct ssl_service_status *status)
 {
 	long cli_st_cb_ret=0, svr_st_cb_ret=0;
 	char temp_key[2048];
@@ -360,30 +387,33 @@ void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct s
 	{
 		return;
 	}
-	char * addr_str= tfe_stream_addr_to_str(addr);
+    char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr);
     struct ssl_service_write_args write_args={svc_cache, status};
 	if(status->is_mutual_auth||status->cli_pinning_status!=PINNING_ST_NOT_PINNING||status->has_protocol_errors)
 	{
 		memset(hash_key, 0, sizeof(hash_key));
-		temp_key_sz = ssl_svc_client_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+        temp_key_sz = ssl_svc_client_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key));
         hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
-		TFE_LOG_DEBUG(g_default_logger, "ssl svc write client table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str);
+        TFE_LOG_DEBUG(g_default_logger, "client table, hash:%s, sni:%s, addr:%s, mutual:%d, pinning:%d, err:%d",
+                      hash_key, chello->sni, addr_str, status->is_mutual_auth, status->cli_pinning_status, status->has_protocol_errors);
         MESA_htable_search_cb(svc_cache->cli_st_hash, hash_key, (unsigned int) hash_key_sz, cli_st_write_cb, &write_args, &cli_st_cb_ret);	
 	}
 	if(status->is_ct||status->is_ev)
 	{	
 		memset(hash_key, 0, sizeof(hash_key));
-		temp_key_sz = ssl_svc_server_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+        temp_key_sz = ssl_svc_server_st_mk_key(chello, tcp_stream->addr, temp_key, sizeof(temp_key));
         hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
-		TFE_LOG_DEBUG(g_default_logger, "ssl svc write server table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str);
+        TFE_LOG_DEBUG(g_default_logger, "server table, hash:%s, sni:%s, addr:%s, ct:%d, ev:%d",
+                      hash_key, chello->sni, addr_str, status->is_ct, status->is_ev);
         MESA_htable_search_cb(svc_cache->srv_st_hash, hash_key, (unsigned int) hash_key_sz, srv_st_write_cb, &write_args, &svr_st_cb_ret);
 	}
 	if(status->is_app_not_pinning)
 	{
 		memset(hash_key, 0, sizeof(hash_key));
-		temp_key_sz = ssl_svc_app_st_mk_key(chello, addr, temp_key, sizeof(temp_key));
+        temp_key_sz = ssl_svc_app_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key));
         hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key;
-		TFE_LOG_DEBUG(g_default_logger, "ssl svc write app table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str);
+        TFE_LOG_DEBUG(g_default_logger, "app table, hash:%s, sni:%s, addr:%s, app_not_pinning:%d",
+					hash_key, chello->sni, addr_str, status->is_app_not_pinning);
         MESA_htable_search_cb(svc_cache->app_st_hash, hash_key, (unsigned int) hash_key_sz, app_st_write_cb, &write_args, &svr_st_cb_ret);
 	}
 

platform/src/ssl_stream.cpp

@@ -43,9 +43,11 @@
 #include <ssl_utils.h>
 #include <ssl_service_cache.h>
 #include <platform.h>
+#include <ssl_ja3.h>
 
 static int SSL_CTX_EX_DATA_IDX_SSLMGR;
 static int SSL_EX_DATA_IDX_SSLSTREAM;
+static unsigned int ssl_ja3_debug;
 
 #define MAX_NET_RETRIES        50
 #define	LATENCY_WARNING_THRESHOLD_MS		1000
@@ -64,7 +66,7 @@ static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0};
  */
 #define DFLT_CURVE "prime256v1"
 
-extern long long certstore_is_unavailable;
+    extern long long certstore_is_unavailable;
 
 enum ssl_stream_stat
 {
@@ -636,6 +638,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 		goto error_out;
 	}
 
+    MESA_load_profile_uint_def(ini_profile, section, "ssl_ja3_debug", &(ssl_ja3_debug), 0);
     MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3");
 	mgr->ssl_min_version = sslver_str2num(version_str);
 
@@ -818,6 +821,15 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg)
 	{
 		case CHELLO_PARSE_SUCCESS:
 		{
+            if (ssl_ja3_debug)
+            {
+                char *addr = tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM);
+                struct ssl_ja3 *fingerprint = ssl_ja3_generate_fingerprint(buf, n);
+                TFE_LOG_INFO(ctx->logger, "ja3:%s, hash:%s addr:%s\n", fingerprint ? fingerprint->ja3 : "null", fingerprint ? fingerprint->ja3_hash : "null", addr);
+                ssl_ja3_free(fingerprint);
+                free(addr);
+			}
+			
             promise_dettach_ctx(promise);
 			ctx->chello=chello;
 			promise_success(promise, ctx);
@@ -890,7 +902,7 @@ int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey)
 	struct ssl_stream* s_upstream=NULL;
 	s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
 	s_upstream->up_parts.svc_status.is_mutual_auth=1;
-	ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream->addr, &s_upstream->up_parts.svc_status);
+	ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream, &s_upstream->up_parts.svc_status);
 	return 0;
 }
 
@@ -1206,18 +1218,18 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr
 			{
 				s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING;
 				ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING);
-				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status);
+				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
 			}
 			else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN)
 			{
 				s_upstream->svc_status.has_protocol_errors=1;
-				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status);
+				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status);
 			}
 			break;
 		case CONN_DIR_UPSTREAM:
 			s_upstream=&(s_stream->up_parts);
 			s_upstream->svc_status.has_protocol_errors=1;
-			ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->up_parts.svc_status));
+			ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->up_parts.svc_status));
 			break;
 		default:
 			assert(0);
@@ -1238,7 +1250,7 @@ void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* m
 	{
 		s_upstream->svc_status.cli_pinning_status=PINNING_ST_MAYBE_PINNING;
 		ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_MAYBE_PINNING);
-		ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->peer->up_parts.svc_status));
+		ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->peer->up_parts.svc_status));
 	}
 	s_stream->error=SSL_STREAM_R_CLIENT_CLOSED;
 	return;
@@ -1311,7 +1323,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
 													error_str, sizeof(error_str), &(s_stream->up_parts.verify_result));
 				s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct;
 				s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev;
-				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &(s_upstream->svc_status));
+				ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &(s_upstream->svc_status));
 				TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d",
 											s_stream->tcp_stream->str_stream_info,
 											s_upstream->client_hello->sni,
@@ -1401,18 +1413,19 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 	clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
 	s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream);
 	svc_status=&s_stream->up_parts.svc_status;
-	ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream->addr, svc_status);
+	ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream, svc_status);
 	if(ret==1)
 	{
 		addr_string=tfe_stream_addr_to_str(s_stream->tcp_stream->addr);
-		TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d",
+		TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d, app_not_pinning:%d",
 									addr_string,
 									chello->sni,
 									svc_status->cli_pinning_status,
 									svc_status->is_mutual_auth,
 									svc_status->has_protocol_errors,
 									svc_status->is_ct,
-									svc_status->is_ev);
+									svc_status->is_ev,
+									svc_status->is_app_not_pinning);
 		free(addr_string);
 		addr_string=NULL;
 	}
@@ -1914,7 +1927,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
 		// struct ssl_service_status svc_status;
 		// memset(&svc_status, 0, sizeof(svc_status));
 		// svc_status.is_app_not_pinning=1;
-		// ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &svc_status);
+		// ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &svc_status);
 		promise_success(p, ctx);
 	}
 
@@ -2022,7 +2035,7 @@ void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, s
 			struct ssl_service_status svc_status;
 			memset(&svc_status, 0, sizeof(svc_status));
 			svc_status.is_app_not_pinning = 1;
-			ssl_service_cache_write(s_stream->mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &svc_status);
+			ssl_service_cache_write(s_stream->mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &svc_status);
 		}
 		const char * sni = (s_stream->up_parts.client_hello && s_stream->up_parts.client_hello->sni) ? s_stream->up_parts.client_hello->sni : "null";
 		TFE_LOG_DEBUG(g_default_logger, "ssl up stream close, rx_offset:%d, sni:%s", rx_offset_this_time, sni);