diff --git a/common/include/tfe_cmsg.h b/common/include/tfe_cmsg.h index e932432..e042def 100644 --- a/common/include/tfe_cmsg.h +++ b/common/include/tfe_cmsg.h @@ -75,6 +75,9 @@ enum tfe_cmsg_tlv_type TFE_CMSG_SRC_IP_LOCATION_CITY, // string max size 256 TFE_CMSG_DST_IP_LOCATION_CITY, // string max size 256 + /* SSL ja3 fingerprint */ + TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, // string max size 32 + /* Add new cmsg here */ /* Add new cmsg here */ /* Add new cmsg here */ diff --git a/conf/tfe/tfe.conf b/conf/tfe/tfe.conf index 46cd3fd..f901274 100644 --- a/conf/tfe/tfe.conf +++ b/conf/tfe/tfe.conf @@ -35,6 +35,7 @@ watchdog_switch=1 watchdog_port=2476 [ssl] +ssl_ja3_debug=0 ssl_max_version=tls13 ssl_min_version=ssl3 ssl_compression=1 diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt index f2813c3..48a4ecb 100644 --- a/platform/CMakeLists.txt +++ b/platform/CMakeLists.txt @@ -3,7 +3,7 @@ find_package(SYSTEMD REQUIRED) add_executable(tfe src/acceptor_kni_v1.cpp src/acceptor_kni_v2.cpp src/ssl_stream.cpp src/key_keeper.cpp src/ssl_fetch_cert.cpp src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp - src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp) + src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/ssl_ja3.cpp) target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external) target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal) diff --git a/platform/include/internal/ssl_ja3.h b/platform/include/internal/ssl_ja3.h new file mode 100644 index 0000000..be14463 --- /dev/null +++ b/platform/include/internal/ssl_ja3.h @@ -0,0 +1,10 @@ +#pragma once + + +struct ssl_ja3 { + char *ja3; + char *ja3_hash; +}; + +struct ssl_ja3 *ssl_ja3_generate_fingerprint(const unsigned char *data, int len); +void ssl_ja3_free(struct ssl_ja3 *fingerprint); \ No newline at end of file diff --git a/platform/include/internal/ssl_service_cache.h b/platform/include/internal/ssl_service_cache.h index 7c5b1de..1a3d2ff 100644 --- a/platform/include/internal/ssl_service_cache.h +++ b/platform/include/internal/ssl_service_cache.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -42,6 +43,6 @@ struct ssl_service_cache struct ssl_service_cache* ssl_service_cache_create(unsigned int slot_size, unsigned int expire_seconds, int fail_as_pinning_cnt, int fail_as_proto_err_cnt, int succ_as_app_not_pinning_cnt, int fail_time_win); void ssl_service_cache_destroy(struct ssl_service_cache* cache); -int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, struct ssl_service_status* result); -void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, const struct ssl_service_status* status); -void ssl_service_cache_stat(struct ssl_service_cache* svc_cache, struct ssl_service_cache_statistics* result); +int ssl_service_cache_read(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, struct ssl_service_status *result); +void ssl_service_cache_write(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, const struct ssl_service_status *status); +void ssl_service_cache_stat(struct ssl_service_cache *svc_cache, struct ssl_service_cache_statistics *result); diff --git a/platform/src/ssl_ja3.cpp b/platform/src/ssl_ja3.cpp new file mode 100644 index 0000000..457c332 --- /dev/null +++ b/platform/src/ssl_ja3.cpp @@ -0,0 +1,603 @@ +#include +#include +#include +#include + +#ifndef MIN +#define MIN(a, b) ((a) > (b) ? (b) : (a)) +#endif + +#define BSB_INIT(b, buffer, size) \ + do \ + { \ + (b).buf = (unsigned char *)buffer; \ + (b).ptr = (unsigned char *)buffer; \ + int s = (int)size; \ + if ((buffer == NULL) || (s < 0)) \ + (b).end = 0; \ + else \ + (b).end = (unsigned char *)buffer + size; \ + } while (0) + +#define BSB_SET_ERROR(b) ((b).end = NULL) +#define BSB_IS_ERROR(b) ((b).end == NULL) +#define BSB_NOT_ERROR(b) ((b).end != NULL) +#define BSB_LENGTH(b) ((b).ptr - (b).buf) +#define BSB_POSITION BSB_LENGTH +#define BSB_SIZE(b) ((b).end - (b).buf) +#define BSB_REMAINING(b) ((b).end ? (b).end - (b).ptr : 0) +#define BSB_WORK_PTR(b) ((b).ptr) + +#define BSB_EXPORT_u08(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 1 <= (b).end) \ + { \ + *(((b).ptr)++) = (unsigned char)x; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_u16(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 2 <= (b).end) \ + { \ + uint16_t t = (uint16_t)x; \ + *(((b).ptr)++) = (t & 0xff00) >> 8; \ + *(((b).ptr)++) = (t & 0x00ff); \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_u32(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 4 <= (b).end) \ + { \ + uint32_t t = x; \ + *(((b).ptr)++) = (t & 0xff000000) >> 24; \ + *(((b).ptr)++) = (t & 0x00ff0000) >> 16; \ + *(((b).ptr)++) = (t & 0x0000ff00) >> 8; \ + *(((b).ptr)++) = (t & 0x000000ff); \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_ptr(b, x, size) \ + do \ + { \ + if ((x || size == 0) && \ + (b).ptr + size <= (b).end && \ + (b).ptr + size >= (b).buf) \ + { \ + memcpy((b).ptr, x, size); \ + (b).ptr += size; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_ptr_some(b, x, size) \ + do \ + { \ + if ((b).ptr + size <= (b).end) \ + { \ + memcpy((b).ptr, x, size); \ + (b).ptr += size; \ + } \ + else if (BSB_NOT_ERROR(b)) \ + { \ + memcpy((b).ptr, x, BSB_REMAINING(b)); \ + (b).ptr += BSB_REMAINING(b); \ + } \ + } while (0) + +#define BSB_EXPORT_cstr(b, x) \ + do \ + { \ + const int size = sizeof x - 1; \ + if ((b).ptr + size <= (b).end) \ + { \ + memcpy((b).ptr, x, size); \ + (b).ptr += size; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_skip(b, size) \ + do \ + { \ + if ((b).ptr + size <= (b).end && \ + (b).ptr + size >= (b).buf) \ + { \ + (b).ptr += size; \ + if ((b).ptr < (b).buf) \ + (b).end = 0; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_EXPORT_rewind(b, size) \ + do \ + { \ + if ((b).ptr - size <= (b).end && \ + (b).ptr - size >= (b).buf) \ + { \ + (b).ptr -= size; \ + if ((b).ptr < (b).buf) \ + (b).end = 0; \ + } \ + else \ + { \ + BSB_SET_ERROR(b); \ + } \ + } while (0) + +#if defined(C9X) + +#define BSB_EXPORT_sprintf(b, ...) \ + do \ + { \ + if ((b).end != 0) \ + { \ + int l = snprintf((char *)(b).ptr, \ + (b).end - (b).ptr, \ + __VA_ARGS__); \ + if (l <= (b).end - (b).ptr) \ + { \ + (b).ptr += l; \ + } \ + else \ + { \ + BSB_SET_ERROR(b); \ + } \ + } \ + } while (0) + +#else + +#define BSB_EXPORT_sprintf(b, args...) \ + do \ + { \ + if ((b).end != 0) \ + { \ + int l = snprintf((char *)(b).ptr, \ + (b).end - (b).ptr, \ + ##args); \ + if (l <= (b).end - (b).ptr) \ + { \ + (b).ptr += l; \ + } \ + else \ + { \ + BSB_SET_ERROR(b); \ + } \ + } \ + } while (0) +#endif + +#define BSB_IMPORT_u08(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 1 <= (b).end) \ + { \ + x = *(((b).ptr)++); \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_IMPORT_u16(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 2 <= (b).end) \ + { \ + x = ((uint16_t)((b).ptr)[0]) << 8 | \ + ((uint16_t)((b).ptr)[1]); \ + (b).ptr += 2; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_IMPORT_u24(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 3 <= (b).end) \ + { \ + x = ((uint32_t)((b).ptr)[0]) << 16 | \ + ((uint32_t)((b).ptr)[1]) << 8 | \ + ((uint32_t)((b).ptr)[2]); \ + (b).ptr += 3; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_IMPORT_u32(b, x) \ + do \ + { \ + if ((b).ptr && (b).ptr + 4 <= (b).end) \ + { \ + x = ((uint32_t)((b).ptr)[0]) << 24 | \ + ((uint32_t)((b).ptr)[1]) << 16 | \ + ((uint32_t)((b).ptr)[2]) << 8 | \ + ((uint32_t)((b).ptr)[3]); \ + (b).ptr += 4; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_LEXPORT_u08(b, x) BSB_EXPORT_u08(b, x) + +#define BSB_LEXPORT_u16(b, x) \ + do \ + { \ + if ((b).ptr + 2 <= (b).end) \ + { \ + uint16_t t = (uint16_t)x; \ + *(((b).ptr)++) = (t & 0x00ff); \ + *(((b).ptr)++) = (t & 0xff00) >> 8; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_LEXPORT_u32(b, x) \ + do \ + { \ + if ((b).ptr + 4 <= (b).end) \ + { \ + uint32_t t = x; \ + *(((b).ptr)++) = (t & 0x000000ff); \ + *(((b).ptr)++) = (t & 0x0000ff00) >> 8; \ + *(((b).ptr)++) = (t & 0x00ff0000) >> 16; \ + *(((b).ptr)++) = (t & 0xff000000) >> 24; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_LIMPORT_u08(b, x) BSB_IMPORT_u08(b, x) + +#define BSB_LIMPORT_u16(b, x) \ + do \ + { \ + if ((b).ptr + 2 <= (b).end) \ + { \ + x = ((uint16_t)((b).ptr)[1]) << 8 | \ + ((uint16_t)((b).ptr)[0]); \ + (b).ptr += 2; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_LIMPORT_u24(b, x) \ + do \ + { \ + if ((b).ptr + 3 <= (b).end) \ + { \ + x = ((uint32_t)((b).ptr)[2]) << 16 | \ + ((uint32_t)((b).ptr)[1]) << 8 | \ + ((uint32_t)((b).ptr)[0]); \ + (b).ptr += 3; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_LIMPORT_u32(b, x) \ + do \ + { \ + if ((b).ptr + 4 <= (b).end) \ + { \ + x = ((uint32_t)((b).ptr)[3]) << 24 | \ + ((uint32_t)((b).ptr)[2]) << 16 | \ + ((uint32_t)((b).ptr)[1]) << 8 | \ + ((uint32_t)((b).ptr)[0]); \ + (b).ptr += 4; \ + } \ + else \ + BSB_SET_ERROR(b); \ + } while (0) + +#define BSB_IMPORT_ptr(b, x, size) \ + do \ + { \ + if ((b).ptr + size <= (b).end && \ + (b).ptr + size >= (b).buf) \ + { \ + (x) = (b).ptr; \ + (b).ptr += size; \ + } \ + else \ + { \ + BSB_SET_ERROR(b); \ + x = 0; \ + } \ + } while (0) + +#define BSB_LIMPORT_ptr BSB_IMPORT_ptr +#define BSB_IMPORT_skip BSB_EXPORT_skip +#define BSB_LIMPORT_skip BSB_EXPORT_skip +#define BSB_IMPORT_rewind BSB_EXPORT_rewind +#define BSB_LIMPORT_rewind BSB_EXPORT_rewind + +#define BSB_memchr(b, ch, pos) \ + do \ + { \ + if (BSB_IS_ERROR(b)) \ + { \ + pos = 0; \ + break; \ + } \ + char *s = memchr((char *)b.ptr, ch, BSB_REMAINING(b)); \ + if (s) \ + pos = (char *)s - (char *)b.ptr; \ + else \ + pos = 0; \ + } while (0) + +#define BSB_memcmp(str, b, len) ((b).ptr + len <= (b).end ? memcmp(str, b.ptr, len) : -1) + +#define BSB_PEEK(b) ((b).ptr + 1 <= (b).end ? *b.ptr : -1) + +#define BSB_IMPORT_zbyte(b, x, size) \ + do \ + { \ + if ((b).ptr + size <= (b).end) \ + { \ + memcpy(x, b, size); \ + (x)[size] = 0; \ + (b).ptr += size; \ + } \ + else \ + { \ + BSB_SET_ERROR(b); \ + (x)[0] = 0; \ + } \ + } while (0) + +/* Private data structure */ +typedef struct bsb +{ + unsigned char *buf; + unsigned char *ptr; + unsigned char *end; +} BSB; + +// https://tools.ietf.org/html/draft-davidben-tls-grease-00 +static int tls_is_grease_value(uint32_t val) +{ + if ((val & 0x0f) != 0x0a) + return 0; + + if ((val & 0xff) != ((val >> 8) & 0xff)) + return 0; + + return 1; +} + +static int md5sum(const char *str, int len, char *buf, int size) +{ + int n; + int ret = 0; + MD5_CTX ctx; + unsigned char tmp[MD5_DIGEST_LENGTH]; + + MD5_Init(&ctx); + MD5_Update(&ctx, str, len); + MD5_Final(tmp, &ctx); + + for (n = 0; n < MD5_DIGEST_LENGTH; n++) + { + ret += snprintf(buf + ret, size - ret, "%.2x", tmp[n]); + } + + return ret; +} + +void ssl_ja3_free(struct ssl_ja3 *fingerprint) +{ + if (fingerprint) + { + if (fingerprint->ja3) + { + free(fingerprint->ja3); + fingerprint->ja3 = NULL; + } + if (fingerprint->ja3_hash) + { + free(fingerprint->ja3_hash); + fingerprint->ja3_hash = NULL; + } + free(fingerprint); + fingerprint = NULL; + } +} + + /****************************************************************************** + * JA3 generation algorithm refers to tls_process_client() in + * https://github.com/aol/moloch. + ******************************************************************************/ +struct ssl_ja3 *ssl_ja3_generate_fingerprint(const unsigned char *data, int len) +{ + BSB sslbsb; + char ja3[30000]; + BSB ja3bsb; + char ecfja3[1000]; + BSB ecfja3bsb; + char eja3[10000]; + BSB eja3bsb; + char ecja3[10000]; + BSB ecja3bsb; + + BSB_INIT(sslbsb, data, len); + BSB_INIT(ja3bsb, ja3, sizeof(ja3)); + BSB_INIT(ecja3bsb, ecja3, sizeof(ecja3)); + BSB_INIT(ecfja3bsb, ecfja3, sizeof(ecfja3)); + BSB_INIT(eja3bsb, eja3, sizeof(eja3)); + + if (BSB_REMAINING(sslbsb) > 5) + { + unsigned char *ssldata = BSB_WORK_PTR(sslbsb); + int ssllen = MIN(BSB_REMAINING(sslbsb) - 5, ssldata[3] << 8 | ssldata[4]); + + BSB pbsb; + BSB_INIT(pbsb, ssldata + 5, ssllen); + + if (BSB_REMAINING(pbsb) > 7) + { + unsigned char *pdata = BSB_WORK_PTR(pbsb); + int plen = MIN(BSB_REMAINING(pbsb) - 4, pdata[2] << 8 | pdata[3]); + + uint16_t ver = 0; + BSB_IMPORT_skip(pbsb, 4); // type + len + BSB_IMPORT_u16(pbsb, ver); + + BSB_EXPORT_sprintf(ja3bsb, "%d,", ver); + + BSB cbsb; + BSB_INIT(cbsb, pdata + 6, plen - 2); // The - 4 for plen is done above, confusing + + if (BSB_REMAINING(cbsb) > 32) + { + BSB_IMPORT_skip(cbsb, 32); // Random + + int skiplen = 0; + BSB_IMPORT_u08(cbsb, skiplen); // Session Id Length + if (skiplen > 0 && BSB_REMAINING(cbsb) > skiplen) + { + // Remove unnecessary code + } + BSB_IMPORT_skip(cbsb, skiplen); // Session Id + + BSB_IMPORT_u16(cbsb, skiplen); // Ciper Suites Length + while (BSB_NOT_ERROR(cbsb) && skiplen > 0) + { + uint16_t c = 0; + BSB_IMPORT_u16(cbsb, c); + if (!tls_is_grease_value(c)) + { + BSB_EXPORT_sprintf(ja3bsb, "%d-", c); + } + skiplen -= 2; + } + BSB_EXPORT_rewind(ja3bsb, 1); // Remove last - + BSB_EXPORT_u08(ja3bsb, ','); + + BSB_IMPORT_u08(cbsb, skiplen); // Compression Length + BSB_IMPORT_skip(cbsb, skiplen); // Compressions + + if (BSB_REMAINING(cbsb) > 6) + { + int etotlen = 0; + BSB_IMPORT_u16(cbsb, etotlen); // Extensions Length + + etotlen = MIN(etotlen, BSB_REMAINING(cbsb)); + + BSB ebsb; + BSB_INIT(ebsb, BSB_WORK_PTR(cbsb), etotlen); + + while (BSB_REMAINING(ebsb) > 4) + { + uint16_t etype = 0, elen = 0; + + BSB_IMPORT_u16(ebsb, etype); + BSB_IMPORT_u16(ebsb, elen); + + if (!tls_is_grease_value(etype)) + BSB_EXPORT_sprintf(eja3bsb, "%d-", etype); + + if (elen > BSB_REMAINING(ebsb)) + break; + + if (etype == 0) + { // SNI + BSB snibsb; + BSB_INIT(snibsb, BSB_WORK_PTR(ebsb), elen); + BSB_IMPORT_skip(ebsb, elen); + + int sni = 0; + BSB_IMPORT_u16(snibsb, sni); // list len + if (sni != BSB_REMAINING(snibsb)) + continue; + + BSB_IMPORT_u08(snibsb, sni); // type + if (sni != 0) + continue; + + BSB_IMPORT_u16(snibsb, sni); // len + if (sni != BSB_REMAINING(snibsb)) + continue; + } + else if (etype == 0x000a) + { // Elliptic Curves + BSB bsb; + BSB_INIT(bsb, BSB_WORK_PTR(ebsb), elen); + BSB_IMPORT_skip(ebsb, elen); + + uint16_t llen = 0; + BSB_IMPORT_u16(bsb, llen); // list len + while (llen > 0 && !BSB_IS_ERROR(bsb)) + { + uint16_t c = 0; + BSB_IMPORT_u16(bsb, c); + if (!tls_is_grease_value(c)) + { + BSB_EXPORT_sprintf(ecja3bsb, "%d-", c); + } + llen -= 2; + } + BSB_EXPORT_rewind(ecja3bsb, 1); // Remove last - + } + else if (etype == 0x000b) + { // Elliptic Curves point formats + BSB bsb; + BSB_INIT(bsb, BSB_WORK_PTR(ebsb), elen); + BSB_IMPORT_skip(ebsb, elen); + + uint16_t llen = 0; + BSB_IMPORT_u08(bsb, llen); // list len + while (llen > 0 && !BSB_IS_ERROR(bsb)) + { + uint8_t c = 0; + BSB_IMPORT_u08(bsb, c); + BSB_EXPORT_sprintf(ecfja3bsb, "%d-", c); + llen -= 1; + } + BSB_EXPORT_rewind(ecfja3bsb, 1); // Remove last - + } + else + { + BSB_IMPORT_skip(ebsb, elen); + } + } + BSB_EXPORT_rewind(eja3bsb, 1); // Remove last - + } + } + } + BSB_IMPORT_skip(sslbsb, ssllen + 5); + + if (BSB_LENGTH(ja3bsb) > 0 && BSB_NOT_ERROR(ja3bsb) && BSB_NOT_ERROR(ecja3bsb) && BSB_NOT_ERROR(eja3bsb) && BSB_NOT_ERROR(ecfja3bsb)) + { + BSB_EXPORT_sprintf(ja3bsb, "%.*s,%.*s,%.*s", (int)BSB_LENGTH(eja3bsb), eja3, (int)BSB_LENGTH(ecja3bsb), ecja3, (int)BSB_LENGTH(ecfja3bsb), ecfja3); + char md5[64] = {0}; + int len = md5sum(ja3, BSB_LENGTH(ja3bsb), md5, sizeof(md5)); + struct ssl_ja3 *result = (struct ssl_ja3 *)calloc(sizeof(struct ssl_ja3), 1); + asprintf(&(result->ja3), "%.*s", (int)BSB_LENGTH(ja3bsb), ja3); + asprintf(&(result->ja3_hash), "%.*s", len, md5); + return result; + } + } + + return NULL; +} \ No newline at end of file diff --git a/platform/src/ssl_service_cache.cpp b/platform/src/ssl_service_cache.cpp index f4c9690..1327a16 100644 --- a/platform/src/ssl_service_cache.cpp +++ b/platform/src/ssl_service_cache.cpp @@ -86,14 +86,38 @@ static size_t ssl_svc_server_st_mk_key(const struct ssl_chello* chello, const s free(addr_str); return key_len; } -static size_t ssl_svc_app_st_mk_key(const struct ssl_chello* chello, const struct tfe_stream_addr * addr, char* key_buff, size_t sz) +static size_t ssl_svc_app_st_mk_key(const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, char *key_buff, size_t sz) { - size_t key_len=0; - const char* sip=NULL, *sport=NULL, *dip=NULL, *dport=NULL; - char * addr_str= tfe_stream_addr_to_str(addr); - tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport); + char ja3_val[64] = {0}; + uint16_t ja3_len = 0; + size_t key_len = 0; + const char *sip = NULL, *sport = NULL, *dip = NULL, *dport = NULL; + char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr); + struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(tcp_stream); + if (cmsg != NULL) + { + int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, (unsigned char *)ja3_val, sizeof(ja3_val), &ja3_len); + if (ret != 0) + { + TFE_LOG_ERROR(g_default_logger, "failed at fetch ssl client ja3 fingerprint from cmsg: %s, %s", strerror(-ret), addr_str); + } + else + { + TFE_LOG_DEBUG(g_default_logger, "fetch ssl client ja3 fingerprint:%s addr: %s", ja3_val, addr_str); + } + } + tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport); - key_len=snprintf(key_buff, sz, "%d:%d:%s:%s", chello->min_version.ossl_format, + // If ja3 is successfully obtained, use ja3 to generate hashkey + if (strlen(ja3_val)) + { + key_len = snprintf(key_buff, sz, "%s:%s", ja3_val, chello->sni ? chello->sni : dip); + free(addr_str); + return key_len; + } + + // otherwise, splicing ssl attributes to generate hashkey + key_len=snprintf(key_buff, sz, "%d:%d:%s:%s", chello->min_version.ossl_format, chello->max_version.ossl_format, chello->sni?chello->sni: dip , chello->alpn?chello->alpn:"null"); @@ -115,18 +139,18 @@ static size_t ssl_svc_app_st_mk_key(const struct ssl_chello* chello, const stru free(addr_str); return key_len; } -static size_t ssl_svc_client_st_mk_key(const struct ssl_chello* chello, const struct tfe_stream_addr * addr, char* key_buff, size_t sz) +static size_t ssl_svc_client_st_mk_key(const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, char *key_buff, size_t sz) { size_t key_len=0; const char* sip=NULL, *sport=NULL, *dip=NULL, *dport=NULL; - char * addr_str= tfe_stream_addr_to_str(addr); - tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport); + char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr); + tfe_stream_addr_str_split(addr_str, &sip, &sport, &dip, &dport); char chello_id_buff[sz]; size_t chello_id_len=0; key_len=snprintf(key_buff, sz, "%s:", sip); - chello_id_len=ssl_svc_app_st_mk_key(chello, addr, chello_id_buff, sizeof(chello_id_buff)); - memcpy(key_buff+key_len, chello_id_buff, MIN(chello_id_len, sz-key_len)); + chello_id_len = ssl_svc_app_st_mk_key(chello, tcp_stream, chello_id_buff, sizeof(chello_id_buff)); + memcpy(key_buff+key_len, chello_id_buff, MIN(chello_id_len, sz-key_len)); key_len += MIN(chello_id_len, sz-key_len); free(addr_str); @@ -305,7 +329,7 @@ static long app_st_write_cb(void * data, const uchar * key, uint size, void * us return 1; } -int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, struct ssl_service_status* result) +int ssl_service_cache_read(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, struct ssl_service_status *result) { long cli_st_cb_ret=0, svr_st_cb_ret=0, app_st_cb_ret=0; char temp_key[2048]; @@ -317,26 +341,29 @@ int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl return 0; } - char * addr_str= tfe_stream_addr_to_str(addr); + char * addr_str= tfe_stream_addr_to_str(tcp_stream->addr); memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_client_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + temp_key_sz = ssl_svc_client_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; MESA_htable_search_cb(svc_cache->cli_st_hash, hash_key, (unsigned int) hash_key_sz, cli_st_read_cb, result, &cli_st_cb_ret); - TFE_LOG_DEBUG(g_default_logger, "ssl svc read client table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, cli_st_cb_ret, chello->sni, addr_str); + TFE_LOG_DEBUG(g_default_logger, "client table, hash:%s, found:%d, sni:%s, addr:%s, mutual:%d, pinning:%d, err:%d", + hash_key, cli_st_cb_ret, chello->sni, addr_str, result->is_mutual_auth, result->cli_pinning_status, result->has_protocol_errors); memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_server_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + temp_key_sz = ssl_svc_server_st_mk_key(chello, tcp_stream->addr, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; MESA_htable_search_cb(svc_cache->srv_st_hash, hash_key, (unsigned int) hash_key_sz, srv_st_read_cb, result, &svr_st_cb_ret); - TFE_LOG_DEBUG(g_default_logger, "ssl svc read server table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, svr_st_cb_ret, chello->sni, addr_str); + TFE_LOG_DEBUG(g_default_logger, "server table, hash:%s, found:%d, sni:%s, addr:%s, ct:%d, ev:%d", + hash_key, svr_st_cb_ret, chello->sni, addr_str, result->is_ct, result->is_ev); memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_app_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + temp_key_sz = ssl_svc_app_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; MESA_htable_search_cb(svc_cache->app_st_hash, hash_key, (unsigned int) hash_key_sz, app_st_read_cb, result, &app_st_cb_ret); - TFE_LOG_DEBUG(g_default_logger, "ssl svc read app table, hash:%s, found:%d, sni:%s, addr:%s", hash_key, app_st_cb_ret, chello->sni, addr_str); + TFE_LOG_DEBUG(g_default_logger, "app table, hash:%s, found:%d, sni:%s, addr:%s, app_not_pinning:%d", + hash_key, app_st_cb_ret, chello->sni, addr_str, result->is_app_not_pinning); - free(addr_str); + free(addr_str); if(cli_st_cb_ret||svr_st_cb_ret||app_st_cb_ret) { @@ -348,7 +375,7 @@ int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl } } -void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct tfe_stream_addr * addr, const struct ssl_service_status* status) +void ssl_service_cache_write(struct ssl_service_cache *svc_cache, const struct ssl_chello *chello, const struct tfe_stream *tcp_stream, const struct ssl_service_status *status) { long cli_st_cb_ret=0, svr_st_cb_ret=0; char temp_key[2048]; @@ -360,31 +387,34 @@ void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct s { return; } - char * addr_str= tfe_stream_addr_to_str(addr); - struct ssl_service_write_args write_args={svc_cache, status}; + char *addr_str = tfe_stream_addr_to_str(tcp_stream->addr); + struct ssl_service_write_args write_args={svc_cache, status}; if(status->is_mutual_auth||status->cli_pinning_status!=PINNING_ST_NOT_PINNING||status->has_protocol_errors) { memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_client_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; - TFE_LOG_DEBUG(g_default_logger, "ssl svc write client table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str); - MESA_htable_search_cb(svc_cache->cli_st_hash, hash_key, (unsigned int) hash_key_sz, cli_st_write_cb, &write_args, &cli_st_cb_ret); + temp_key_sz = ssl_svc_client_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + TFE_LOG_DEBUG(g_default_logger, "client table, hash:%s, sni:%s, addr:%s, mutual:%d, pinning:%d, err:%d", + hash_key, chello->sni, addr_str, status->is_mutual_auth, status->cli_pinning_status, status->has_protocol_errors); + MESA_htable_search_cb(svc_cache->cli_st_hash, hash_key, (unsigned int) hash_key_sz, cli_st_write_cb, &write_args, &cli_st_cb_ret); } if(status->is_ct||status->is_ev) { memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_server_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; - TFE_LOG_DEBUG(g_default_logger, "ssl svc write server table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str); - MESA_htable_search_cb(svc_cache->srv_st_hash, hash_key, (unsigned int) hash_key_sz, srv_st_write_cb, &write_args, &svr_st_cb_ret); + temp_key_sz = ssl_svc_server_st_mk_key(chello, tcp_stream->addr, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + TFE_LOG_DEBUG(g_default_logger, "server table, hash:%s, sni:%s, addr:%s, ct:%d, ev:%d", + hash_key, chello->sni, addr_str, status->is_ct, status->is_ev); + MESA_htable_search_cb(svc_cache->srv_st_hash, hash_key, (unsigned int) hash_key_sz, srv_st_write_cb, &write_args, &svr_st_cb_ret); } if(status->is_app_not_pinning) { memset(hash_key, 0, sizeof(hash_key)); - temp_key_sz = ssl_svc_app_st_mk_key(chello, addr, temp_key, sizeof(temp_key)); - hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; - TFE_LOG_DEBUG(g_default_logger, "ssl svc write app table, hash:%s, sni:%s, addr:%s", hash_key, chello->sni, addr_str); - MESA_htable_search_cb(svc_cache->app_st_hash, hash_key, (unsigned int) hash_key_sz, app_st_write_cb, &write_args, &svr_st_cb_ret); + temp_key_sz = ssl_svc_app_st_mk_key(chello, tcp_stream, temp_key, sizeof(temp_key)); + hash_key_sz = tfe_hexdump(hash_key, (unsigned char *)temp_key, temp_key_sz) - hash_key; + TFE_LOG_DEBUG(g_default_logger, "app table, hash:%s, sni:%s, addr:%s, app_not_pinning:%d", + hash_key, chello->sni, addr_str, status->is_app_not_pinning); + MESA_htable_search_cb(svc_cache->app_st_hash, hash_key, (unsigned int) hash_key_sz, app_st_write_cb, &write_args, &svr_st_cb_ret); } free(addr_str); diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp index 1b151a4..4b70c8d 100644 --- a/platform/src/ssl_stream.cpp +++ b/platform/src/ssl_stream.cpp @@ -43,9 +43,11 @@ #include #include #include +#include static int SSL_CTX_EX_DATA_IDX_SSLMGR; static int SSL_EX_DATA_IDX_SSLSTREAM; +static unsigned int ssl_ja3_debug; #define MAX_NET_RETRIES 50 #define LATENCY_WARNING_THRESHOLD_MS 1000 @@ -64,7 +66,7 @@ static unsigned char SSL_ALPN_HTTP_2[]={2, 'h','2',0}; */ #define DFLT_CURVE "prime256v1" -extern long long certstore_is_unavailable; + extern long long certstore_is_unavailable; enum ssl_stream_stat { @@ -636,7 +638,8 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section goto error_out; } - MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3"); + MESA_load_profile_uint_def(ini_profile, section, "ssl_ja3_debug", &(ssl_ja3_debug), 0); + MESA_load_profile_string_def(ini_profile, section, "ssl_min_version", version_str, sizeof(version_str), "ssl3"); mgr->ssl_min_version = sslver_str2num(version_str); if (mgr->ssl_min_version < 0) @@ -818,7 +821,16 @@ static void peek_client_hello_cb(evutil_socket_t fd, short what, void * arg) { case CHELLO_PARSE_SUCCESS: { - promise_dettach_ctx(promise); + if (ssl_ja3_debug) + { + char *addr = tfe_string_addr_create_by_fd(fd, CONN_DIR_DOWNSTREAM); + struct ssl_ja3 *fingerprint = ssl_ja3_generate_fingerprint(buf, n); + TFE_LOG_INFO(ctx->logger, "ja3:%s, hash:%s addr:%s\n", fingerprint ? fingerprint->ja3 : "null", fingerprint ? fingerprint->ja3_hash : "null", addr); + ssl_ja3_free(fingerprint); + free(addr); + } + + promise_dettach_ctx(promise); ctx->chello=chello; promise_success(promise, ctx); peek_client_hello_ctx_free(ctx); @@ -890,7 +902,7 @@ int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey) struct ssl_stream* s_upstream=NULL; s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM); s_upstream->up_parts.svc_status.is_mutual_auth=1; - ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream->addr, &s_upstream->up_parts.svc_status); + ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, s_upstream->tcp_stream, &s_upstream->up_parts.svc_status); return 0; } @@ -1206,18 +1218,18 @@ void ssl_stream_process_error(struct ssl_stream * s_stream, unsigned long sslerr { s_upstream->svc_status.cli_pinning_status=PINNING_ST_PINNING; ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_PINNING); - ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status); + ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status); } else if(sslerr>0 && sslerr!=SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN) { s_upstream->svc_status.has_protocol_errors=1; - ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &s_upstream->svc_status); + ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &s_upstream->svc_status); } break; case CONN_DIR_UPSTREAM: s_upstream=&(s_stream->up_parts); s_upstream->svc_status.has_protocol_errors=1; - ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->up_parts.svc_status)); + ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->up_parts.svc_status)); break; default: assert(0); @@ -1238,7 +1250,7 @@ void ssl_stream_process_zero_eof(struct ssl_stream * s_stream, struct ssl_mgr* m { s_upstream->svc_status.cli_pinning_status=PINNING_ST_MAYBE_PINNING; ssl_stream_set_cmsg_integer(s_stream, TFE_CMSG_SSL_PINNING_STATE, PINNING_ST_MAYBE_PINNING); - ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream->addr, &(s_stream->peer->up_parts.svc_status)); + ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, s_stream->tcp_stream, &(s_stream->peer->up_parts.svc_status)); } s_stream->error=SSL_STREAM_R_CLIENT_CLOSED; return; @@ -1311,7 +1323,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events, error_str, sizeof(error_str), &(s_stream->up_parts.verify_result)); s_upstream->svc_status.is_ct=s_upstream->verify_result.is_ct; s_upstream->svc_status.is_ev=s_upstream->verify_result.is_ev; - ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &(s_upstream->svc_status)); + ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &(s_upstream->svc_status)); TFE_LOG_DEBUG(mgr->logger, "stream:%s sni:%s hostmatch:%d, ct:%d, ev:%d", s_stream->tcp_stream->str_stream_info, s_upstream->client_hello->sni, @@ -1401,18 +1413,19 @@ static void peek_chello_on_succ(future_result_t * result, void * user) clock_gettime(CLOCK_MONOTONIC, &(ctx->start)); s_stream= ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL, ctx->tcp_stream); svc_status=&s_stream->up_parts.svc_status; - ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream->addr, svc_status); + ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, s_stream->tcp_stream, svc_status); if(ret==1) { addr_string=tfe_stream_addr_to_str(s_stream->tcp_stream->addr); - TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d", + TFE_LOG_DEBUG(ctx->mgr->logger, "%s %s service status pinning:%d, mauth:%d, err:%d, ct:%d, ev:%d, app_not_pinning:%d", addr_string, chello->sni, svc_status->cli_pinning_status, svc_status->is_mutual_auth, svc_status->has_protocol_errors, svc_status->is_ct, - svc_status->is_ev); + svc_status->is_ev, + svc_status->is_app_not_pinning); free(addr_string); addr_string=NULL; } @@ -1914,7 +1927,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events, // struct ssl_service_status svc_status; // memset(&svc_status, 0, sizeof(svc_status)); // svc_status.is_app_not_pinning=1; - // ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream->addr, &svc_status); + // ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, s_stream->tcp_stream, &svc_status); promise_success(p, ctx); } @@ -2022,7 +2035,7 @@ void ssl_stream_free(struct ssl_stream * s_stream, struct event_base * evbase, s struct ssl_service_status svc_status; memset(&svc_status, 0, sizeof(svc_status)); svc_status.is_app_not_pinning = 1; - ssl_service_cache_write(s_stream->mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream->addr, &svc_status); + ssl_service_cache_write(s_stream->mgr->svc_cache, s_stream->up_parts.client_hello, s_stream->tcp_stream, &svc_status); } const char * sni = (s_stream->up_parts.client_hello && s_stream->up_parts.client_hello->sni) ? s_stream->up_parts.client_hello->sni : "null"; TFE_LOG_DEBUG(g_default_logger, "ssl up stream close, rx_offset:%d, sni:%s", rx_offset_this_time, sni);