fix compile warning

CMakeLists.txt

@@ -31,8 +31,11 @@ elseif(ENABLE_SANITIZE_THREAD)
 endif()
 # end of for ASAN
 
+include_directories(${PROJECT_SOURCE_DIR}/third_party/cJSON/)
+include_directories(${PROJECT_SOURCE_DIR}/third_party/uthash/)
 include_directories(${PROJECT_SOURCE_DIR}/src/)
 
+aux_source_directory(${PROJECT_SOURCE_DIR}/third_party/cJSON/ SRCLIST)
 aux_source_directory(${PROJECT_SOURCE_DIR}/src SRCLIST)
 
 # Shared Library Output
@@ -73,7 +76,6 @@ target_link_libraries(${lib_name}_test ${lib_name}_static)
 
 enable_testing()
 
-add_test(NAME sample COMMAND ${lib_name}_sample)
 add_test(NAME test COMMAND ${lib_name}_test -f ../fp.json -t ../data.json)
 
 include(Package)

Makefile.am

@@ -1,2 +0,0 @@
-SUBDIRS = src example test
-ACLOCAL_AMFLAGS=-I m4

example/osfp_example.c

@@ -25,6 +25,7 @@
 
 #define VLAN_MAX_LAYER  2
 
+struct osfp_db *g_osfp_db;
 
 /* Port is just a uint16_t */
 typedef uint16_t Port;
@@ -431,7 +432,7 @@ const char *PrintInet(int af, const void *src, char *dst, socklen_t size)
 void example_detect(struct osfp_db *osfp_db, Packet *p)
 {
     int ret;
-    char str_buf[1024];
+    char str_buf[2048] = "";
     //unsigned char *iph = (unsigned char *)(p->iph != NULL ? (void *)p->iph : (void *)p->ip6h);
     struct iphdr *iph;
     struct ip6_hdr *ip6h;
@@ -448,6 +449,15 @@ void example_detect(struct osfp_db *osfp_db, Packet *p)
     osfp_profile_cycle(c2);
 
 
+    struct osfp_fingerprint fp = {0};
+    if (iph) {
+        osfp_fingerprinting((unsigned char*)iph, (unsigned char*)tcph, tcph_len, &fp, 4);
+    } else if (ip6h) {
+        osfp_fingerprinting((unsigned char*)iph, (unsigned char*)tcph, tcph_len, &fp, 6);
+    } else {
+        goto exit;
+    }
+
     osfp_profile_get_cycle(c1);
     if (iph) {
         result = osfp_ipv4_identify(osfp_db, iph, tcph, tcph_len);
@@ -471,14 +481,24 @@ void example_detect(struct osfp_db *osfp_db, Packet *p)
 
     char *json = osfp_result_score_detail_export(result);
 
-    if (1) {
+    if (debug_enable) {
+        if (p->tcph->ack) {
+            printf("---------------------------  SYN/ACK\n");
+        } else {
+            printf("---------------------------  SYN\n");
+        }
+        osfp_fingerprint_to_json_buf(&fp, str_buf, 2048, 0);
+        printf("%s\n", str_buf);
         printf("Example ipv4 header detect: --------------------------\n");
         printf("Connection info: %s:%d -> %s:%d\n", p->srcip, p->sp, p->dstip, p->dp);
         printf("Most likely os class: %s\n", osfp_result_os_name_get(result));
         printf("Details:\n");
         printf("%s\n", json);
+        fflush(stdout);
     }
 
+    free(json);
+
 exit:
     if (result) {
         osfp_result_free(result);
@@ -503,11 +523,6 @@ void process_packet(char *user, struct pcap_pkthdr *h, u_char *pkt)
         goto exit;
     }
 
-    if (p->tcph->ack) {
-        printf("---------------------------  SYN/ACK\n");
-    } else {
-        printf("---------------------------  SYN\n");
-    }
 
     if (p->iph) {
         PrintInet(AF_INET, (const void *)&(p->src.addr_data32[0]), p->srcip, sizeof(p->srcip));
@@ -519,7 +534,7 @@ void process_packet(char *user, struct pcap_pkthdr *h, u_char *pkt)
 
     // tcp/ip header detect example for user
     int i;
-    for (i = 0; i < 1; i++) {
+    for (i = 0; i < 1000; i++) {
         example_detect(osfp_db, p);
     }
 
@@ -548,6 +563,10 @@ static void signal_handler(int signum)
         printf("%s: %u\n", osfp_os_class_id_to_name(i), result_os_count[i]);
     }
 
+    fflush(stdout);
+
+    osfp_db_free(g_osfp_db);
+
     exit(0);
 }
 
@@ -676,6 +695,8 @@ int main(int argc, char *argv[])
         exit(1);
     }
 
+    g_osfp_db = osfp_db;
+
     // loop
     while (1) {
         int r = pcap_dispatch(pcap_handle, 0, (pcap_handler)process_packet, (void*)osfp_db);    

example/sample.c

@@ -1,4 +1,5 @@
-#include "stdio.h" 
+#include <stdio.h>
+#include <stdlib.h>
 #include "osfp.h" 
  
 char iph[] = { 
@@ -17,6 +18,7 @@ char tcph[] = {
 int main(int argc, char **argv) 
 { 
     const char *json_file_path = "./fp.json";
+    char *detail_json;
  
     struct iphdr *l3_hdr = (struct iphdr *)iph; 
     struct tcphdr *l4_hdr = (struct tcphdr *)tcph; 
@@ -27,7 +29,9 @@ int main(int argc, char **argv)
         struct osfp_result *result = osfp_ipv4_identify(db, l3_hdr, l4_hdr, l4_hdr_len); 
         if (result) { 
             printf("likely os: %s\n", osfp_result_os_name_get(result)); 
-            printf("details: \n%s\n", osfp_result_score_detail_export(result)); 
+            detail_json = osfp_result_score_detail_export(result);
+            printf("details: \n%s\n", detail_json); 
+            free(detail_json);
             osfp_result_free(result);
         } 
         osfp_db_free(db); 

fp.json

@@ -72590,5 +72590,21 @@
     "tcp_flags": 2,
     "ip_tos": 0,
     "os": "Linux"
+  },
+  {
+    "tcp_options": "M1460,N,W3,N,N,T0,S,E,E,",
+    "tcp_options_ordered": "MNWNNTSEE",
+    "ip_total_length": 64,
+    "tcp_off": 44,
+    "tcp_window_scaling": 3,
+    "tcp_window_size": 31744,
+    "ip_ttl": 64,
+    "ip_id": 0,
+    "tcp_timestamp": 1,
+    "tcp_timestamp_echo_reply": 0,
+    "tcp_mss": 1460,
+    "tcp_flags": 2,
+    "ip_tos": 0,
+    "os": "iOS"
   }
 ]

src/osfp.c

@@ -1,3 +1,8 @@
+#include <sys/fcntl.h>
+#include <unistd.h>
+
+#include "cJSON.h"
+
 #include "osfp_common.h"
 
 #include "osfp.h"
@@ -5,71 +10,6 @@
 #include "osfp_score_db.h"
 #include "osfp_log.h"
 
-#define OSFP_DEFAULT_RESULT_BUFLEN_MAX  512
-#define OSFP_LOWEST_SCORE_LIMIT 20
-
-static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score)
-{
-    int i;
-    unsigned int tmp_score;
-    unsigned int sum_score;
-    unsigned int likely_score;
-    enum osfp_os_class_id likely_os_class;
-    struct osfp_result *result;
-
-    result = calloc(1, sizeof(struct osfp_result));
-    if (result == NULL) {
-        goto exit;
-    }
-
-    likely_score = 0;
-    likely_os_class = OSFP_OS_CLASS_OTHERS;
-    sum_score = 0;
-
-    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        tmp_score = os_class_score->scores[i];
-
-        if (likely_score < tmp_score) {
-            likely_score = tmp_score;
-            likely_os_class = i;
-        }
-        result->details[i].score = tmp_score;
-        sum_score += tmp_score;
-    }
-
-    if (sum_score) {
-        for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-            result->details[i].possibility = OSFP_PERCENTILE * result->details[i].score / sum_score;
-        }
-    }
-
-    if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
-        likely_os_class = OSFP_OS_CLASS_OTHERS;
-    } else {
-        for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-            if (likely_os_class == i) {
-                continue;
-            }
-            if (likely_score == os_class_score->scores[i]) {
-                if (likely_os_class == OSFP_OS_CLASS_LINUX && i == OSFP_OS_CLASS_ANDROID) {
-                    continue;
-                } else if (likely_os_class == OSFP_OS_CLASS_MAC_OS && i == OSFP_OS_CLASS_IOS) {
-                    continue;
-                } else {
-                    likely_os_class = OSFP_OS_CLASS_UNKNOWN;
-                    break;
-                }
-            }
-        }
-    }
-
-    result->likely_os_class = likely_os_class;
-
-    return result;
-exit:
-    return NULL;
-}
-
 const char *osfp_result_os_name_get(struct osfp_result *result)
 {
     enum osfp_os_class_id os_class;
@@ -89,24 +29,21 @@ const char *osfp_result_os_name_get(struct osfp_result *result)
 char *osfp_result_score_detail_export(struct osfp_result *result)
 {
     int i;
-    char *result_str = NULL;
+    char *result_str;
-    cJSON *root = NULL;
+    cJSON *root;
     cJSON *array;
     cJSON *os_score;
+    cJSON *matched;
 
     osfp_profile_cycle(c1);
     osfp_profile_cycle(c2);
+
     osfp_profile_get_cycle(c1);
 
     if (result == NULL) {
         goto exit;
     }
 
-    if (result->json_str != NULL) {
-        result_str = result->json_str;
-        goto exit;
-    }
-
     root = cJSON_CreateObject();
     if (root == NULL) {
         goto exit;
@@ -132,99 +69,64 @@ char *osfp_result_score_detail_export(struct osfp_result *result)
         }
     }
 
+    if (result->matched) {
+        matched = cJSON_AddObjectToObject(root, "matched");
+        if (matched) {
+            cJSON_AddStringToObject(matched, "fingerprint", result->matched);
+        }
+    }
+
     result_str = cJSON_Print(root);
     if (result_str == NULL) {
         goto exit;
     }
 
-    result->json_str = result_str;
+    cJSON_Delete(root);
 
+    osfp_profile_get_cycle(c2);
+    osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1);
+
+    return result_str;
 exit:
     if (root) {
         cJSON_Delete(root);
     }
-    if (result_str) {
+    return NULL;
-        osfp_profile_get_cycle(c2);
-        osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1);
-    }
-    return result_str;
 }
 
 void osfp_result_free(struct osfp_result *result)
 {
     if (result) {
-        if (result->json_str) {
-            free(result->json_str);
-        }
         free(result);
     }
 }
 
-struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
+struct osfp_result *osfp_ip_identify(struct osfp_db *db, unsigned char *l3_hdr, unsigned char *l4_hdr, unsigned int l4_hdr_len, unsigned int ip_version)
 {
-    int ret = OSFP_EINVAL;
+    int ret;
-    struct osfp_fingerprint fp;
-    struct osfp_os_class_score os_class_score;
-    struct osfp_result *result;
-
-    osfp_profile_cycle(c1);
-    osfp_profile_cycle(c2);
-
-    if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr == 0) {
-        goto exit;
-    }
-
-    osfp_profile_get_cycle(c1);
-    ret = osfp_fingerprinting((unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, &fp, 4);
-    osfp_profile_get_cycle(c2);
-    osfp_profile_counter_update(&osfp_profile_fingerprinting, c2 - c1);
-    if (ret != 0) {
-        goto exit;
-    }
-
-    osfp_profile_get_cycle(c1);
-    ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
-    osfp_profile_get_cycle(c2);
-    osfp_profile_counter_update(&osfp_profile_score, c2 - c1);
-    if (ret != 0) {
-        goto exit;
-    }
-
-    osfp_profile_get_cycle(c1);
-    result = osfp_result_build(&os_class_score);
-    osfp_profile_get_cycle(c2);
-    osfp_profile_counter_update(&osfp_profile_result_build, c2 - c1);
-    if (result == NULL) {
-        goto exit;
-    }
-
-    return result;
-exit:
-    return NULL;
-}
-
-struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
-{
-    int ret = OSFP_EINVAL;
     struct osfp_fingerprint fp;
     struct osfp_os_class_score os_class_score;
     struct osfp_result *result;
+    const char *matched;
 
     if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr_len == 0) {
         goto exit;
     }
 
-    ret = osfp_fingerprinting((unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, &fp, 6);
+    ret = osfp_fingerprinting(l3_hdr, l4_hdr, l4_hdr_len, &fp, ip_version);
     if (ret != 0) {
         goto exit;
     }
 
+    matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
+    if (matched == NULL) {
         ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
         if (ret != 0) {
             goto exit;
         }
+    }
 
-    result = osfp_result_build(&os_class_score);
+    result = osfp_result_build(&os_class_score, matched);
     if (result == NULL) {
         goto exit;
     }
@@ -234,57 +136,38 @@ exit:
     return NULL;
 }
 
-struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str)
+struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
 {
-    int ret = OSFP_EINVAL;
+    return osfp_ip_identify(db, (unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, 4);
-    struct osfp_fingerprint fp;
-    struct osfp_os_class_score os_class_score;
-    struct osfp_result *result;
-
-    if (db == NULL) {
-        goto exit;
 }
 
-    ret = osfp_fingerprint_from_json(&fp, (char *)json_str);
+struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
-    if (ret != 0) {
+{
-        goto exit;
+    return osfp_ip_identify(db, (unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, 6);
 }
 
-    ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+struct osfp_db *osfp_db_new(const char *fp_path)
-    if (ret != 0) {
-        goto exit;
-    }
-
-    result = osfp_result_build(&os_class_score);
-    if (result == NULL) {
-        goto exit;
-    }
-
-    return result;
-exit:
-    return NULL;
-}
-
-struct osfp_db *osfp_db_new(const char *db_json_file)
 {
     int ret;
     struct osfp_db *db;
 
+    if (fp_path == NULL) {
+        goto exit;
+    }
+
+    if (0 != access(fp_path, R_OK)) {
+        goto exit;
+    }
+
     db = calloc(1, sizeof(struct osfp_db));
     if (db == NULL) {
         goto exit;
     }
 
-    if (db_json_file != NULL) {
+    db->db_json_path = strdup((const char*)fp_path);
-        if (0 != access(db_json_file, R_OK)) {
-            goto exit;
-        }
-
-        db->db_json_path = strdup((const char*)db_json_file);
     if (db->db_json_path == NULL) {
         goto exit;
     }
-    }
 
     db->score_db = (void *)osfp_score_db_create();
     if (db->score_db == NULL) {

src/osfp.h

@@ -32,6 +32,18 @@ struct osfp_db *osfp_db_new(const char *db_json_path);
  */
 void osfp_db_free(struct osfp_db *db);
 
+/**
+ * @brief 通过 IP 头部和 TCP 头部识别操作系统。
+ * 
+ * @param db 操作系统指纹库。
+ * @param l3_hdr 指向 IP 头部的指针。
+ * @param l4_hdr 指向 TCP 头部的指针。
+ * @param l4_hdr_len TCP 头部的长度（注意：包含TCP选项部分）。
+ * @param ip_version IP头版本号，4 或 6）。
+ * @return 指向操作系统识别结果的指针。
+ */
+struct osfp_result *osfp_ip_identify(struct osfp_db *db, unsigned char *l3_hdr, unsigned char *l4_hdr, unsigned int l4_hdr_len, unsigned int ip_version);
+
 /**
  * @brief 通过 IPv4 头部和 TCP 头部识别操作系统。
  * 
@@ -54,15 +66,6 @@ struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr,
  */
 struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len);
 
-/**
- * @brief 通过 json 格式的指纹识别操作系统。
- * 
- * @param db 操作系统指纹库。
- * @param json_str 指纹字符串。
- * @return 指向操作系统识别结果的指针（注意：内存需要使用者释放）。
- */
-struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str);
-
 /**
  * @brief 获取操作系统识别结果的操作系统名称。
  * 

src/osfp_common.c

@@ -1,6 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+
 #include "osfp_common.h"
 
 #include "osfp.h"
+#include "osfp_log.h"
 
 unsigned int osfp_profile_enable;
 
@@ -15,6 +23,7 @@ const char *osfp_os_class_name[OSFP_OS_CLASS_MAX] = {
 };
 
 struct osfp_profile_counter osfp_profile_fingerprinting;
+struct osfp_profile_counter osfp_profile_prefilter;
 struct osfp_profile_counter osfp_profile_score;
 struct osfp_profile_counter osfp_profile_result_build;
 struct osfp_profile_counter osfp_profile_result_export;
@@ -24,6 +33,85 @@ const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class)
     return osfp_os_class_name[os_class];
 }
 
+enum osfp_os_class_id osfp_os_class_name_to_id(char *name)
+{
+    int i, namelen;
+    const char *os_class_name;
+
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        os_class_name = osfp_os_class_id_to_name(i);
+        if (0 == strncmp(name, os_class_name, strlen(os_class_name))) {
+            return (enum osfp_os_class_id)i;
+        }
+    }
+
+    return OSFP_OS_CLASS_MAX;
+}
+
+struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score, const char *matched)
+{
+    int i;
+    unsigned int tmp_score;
+    unsigned int likely_score;
+    enum osfp_os_class_id likely_os_class;
+    struct osfp_result *result;
+
+    result = calloc(1, sizeof(struct osfp_result));
+    if (result == NULL) {
+        goto exit;
+    }
+
+    likely_score = 0;
+    likely_os_class = OSFP_OS_CLASS_OTHERS;
+
+    // likely os score
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        tmp_score = os_class_score->scores[i];
+
+        if (likely_score < tmp_score) {
+            likely_score = tmp_score;
+            likely_os_class = i;
+        }
+        result->details[i].score = tmp_score;
+    }
+
+    // prefiltered
+    if (likely_score == OSFP_PERCENTILE) {
+        goto end;
+    }
+
+    // too low to tell os class
+    if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
+        likely_os_class = OSFP_OS_CLASS_OTHERS;
+        goto end;
+    }
+
+    // when the tied likely scores appear between win/apple-like/unix-like, we throw unknown
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        if (likely_os_class == i) {
+            continue;
+        }
+        if (likely_score != os_class_score->scores[i]) {
+            continue;
+        }
+        if (likely_os_class == OSFP_OS_CLASS_LINUX && i == OSFP_OS_CLASS_ANDROID) {
+            continue;
+        }
+        if (likely_os_class == OSFP_OS_CLASS_MAC_OS && i == OSFP_OS_CLASS_IOS) {
+            continue;
+        }
+        likely_os_class = OSFP_OS_CLASS_UNKNOWN;
+        break;
+    }
+
+end:
+    result->likely_os_class = likely_os_class;
+    result->matched = matched;
+    return result;
+exit:
+    return NULL;
+}
+
 void osfp_profile_counter_print(struct osfp_profile_counter *profile, const char *name)
 {
     printf("profile %s: avg: %lu max: %lu min: %lu curr: %lu total: %lu count: %lu\n",
@@ -57,8 +145,9 @@ void osfp_profile_counter_update(struct osfp_profile_counter *profile, unsigned
 void osfp_profile_print_stats(void)
 {
     osfp_profile_counter_print(&osfp_profile_fingerprinting, "fingerprinting");
+    osfp_profile_counter_print(&osfp_profile_prefilter, "prefilter");
     osfp_profile_counter_print(&osfp_profile_score, "score");
-    osfp_profile_counter_print(&osfp_profile_result_build, "result_build");
+    osfp_profile_counter_print(&osfp_profile_result_build, "result build");
     osfp_profile_counter_print(&osfp_profile_result_export, "result export");
 }
 
@@ -67,24 +156,51 @@ void osfp_profile_set(unsigned int enabled)
     osfp_profile_enable = enabled;
 }
 
-enum osfp_os_class_id osfp_os_class_name_to_id(char *name)
+char *osfp_read_file(char *fp_file)
 {
-    enum osfp_os_class_id os_class;
+    int ret = -1;
+    char *file_buffer = NULL;
+    unsigned int file_len = 0;
+    FILE *fp = NULL;
+    struct stat st;
 
-    if (0 == strncmp(name, OSFP_OS_CLASS_NAME_WINDOWS, strlen(OSFP_OS_CLASS_NAME_WINDOWS))) {
+    if (0 > stat(fp_file, &st)) {
-        os_class = OSFP_OS_CLASS_WINDOWS;
+        osfp_log_error("stat() failed on '%s'.\n", fp_file);
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_LINUX, strlen(OSFP_OS_CLASS_NAME_LINUX))) {
+        goto exit;
-        os_class = OSFP_OS_CLASS_LINUX;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_MAC_OS, strlen(OSFP_OS_CLASS_NAME_MAC_OS))) {
-        os_class = OSFP_OS_CLASS_MAC_OS;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_IOS, strlen(OSFP_OS_CLASS_NAME_IOS))) {
-        os_class = OSFP_OS_CLASS_IOS;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_ANDROID, strlen(OSFP_OS_CLASS_NAME_ANDROID))) {
-        os_class = OSFP_OS_CLASS_ANDROID;
-    } else {
-        os_class = OSFP_OS_CLASS_MAX;
     }
 
-    return os_class;
+    if (st.st_size == 0) { 
+        osfp_log_error("Empty file: '%s'.\n", fp_file);
+        goto exit;
+    }
+
+    file_len = (unsigned int)st.st_size;
+    file_buffer = malloc(file_len + 1);
+    if (file_buffer == NULL) {
+        osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file);
+        goto exit;
+    }
+
+    fp = fopen(fp_file, "r");
+    if (fp == NULL) {
+        osfp_log_error("Cannot open '%s' for reading.\n", fp_file);
+        goto exit;
+    }
+
+    ret = fread(file_buffer, 1, file_len, fp);
+    if (ret < 0) {
+        osfp_log_error("fread() failed on '%s'.\n", fp_file);
+        free(file_buffer);
+        fclose(fp);
+        goto exit;
+    }
+
+    fclose(fp);
+
+    file_buffer[file_len] = 0;
+
+    return file_buffer;
+exit:
+    return NULL;
 }
 

src/osfp_common.h

@@ -1,28 +1,6 @@
 #ifndef __OSFP_COMMON_H__
 #define __OSFP_COMMON_H__
 
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <malloc.h>
-#include <stdarg.h>
-#include <time.h>
-
-#include <sys/fcntl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-
-#include "utarray.h"
-#include "uthash.h"
-#include "utlist.h"
-#include "utringbuffer.h"
-#include "utstack.h"
-#include "utstring.h"
-
-#include "cJSON.h"
-
-#include "osfp.h"
-
 static inline unsigned long long osfp_rdtsc(void)
 {
 	union {
@@ -59,6 +37,7 @@ struct osfp_profile_counter {
 };
 
 extern struct osfp_profile_counter osfp_profile_fingerprinting;
+extern struct osfp_profile_counter osfp_profile_prefilter;
 extern struct osfp_profile_counter osfp_profile_score;
 extern struct osfp_profile_counter osfp_profile_result_build;
 extern struct osfp_profile_counter osfp_profile_result_export;
@@ -157,6 +136,7 @@ enum osfp_os_class_id {
 #define OSFP_OS_CLASS_FLAG_IOS       OSFP_BIT_U32(OSFP_OS_CLASS_IOS)
 #define OSFP_OS_CLASS_FLAG_ANDROID   OSFP_BIT_U32(OSFP_OS_CLASS_ANDROID)
 
+#define OSFP_LOWEST_SCORE_LIMIT 20
 
 enum osfp_error_code {
     OSFP_NOERR,
@@ -169,6 +149,9 @@ enum osfp_error_code {
     OSFP_ERR_FINGERPRINTING_UNSUPPORTED,
 };
 
+struct osfp_os_class_score {
+    unsigned int scores[OSFP_OS_CLASS_MAX];
+};
 
 /**
  * @brief 结构体用于 osfp_result 中的详细结果。
@@ -182,9 +165,9 @@ struct osfp_result_detail {
  * @brief 结构体用于表示操作系统识别结果。
  */
 struct osfp_result {
-    char *json_str;                              // JSON 字符串
     enum osfp_os_class_id likely_os_class;        // 最可能的操作系统类别
     struct osfp_result_detail details[OSFP_OS_CLASS_MAX];   // 详细结果数组
+    const char *matched;                        // 精确匹配到的指纹特征
 };
 
 /**
@@ -196,7 +179,8 @@ struct osfp_db {
 };
 
 enum osfp_os_class_id osfp_os_class_name_to_id(char *name);
-
 const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class);
+struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score, const char *matched);
+char *osfp_read_file(char *fp_file);
 
 #endif

src/osfp_fingerprint.c

@@ -1,3 +1,10 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "cJSON.h"
+
 #include "osfp_common.h"
 
 #include "osfp.h"
@@ -21,6 +28,27 @@
 
 #define OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME           "OSFP_UNKNOWN"
 
+#define OSFP_FP_SET_FIELD(fp, field_id, v, l) do {                                    \
+        (fp)->fields[(field_id)].name = osfp_fingerprint_get_field_name(field_id);  \
+        (fp)->fields[(field_id)].enabled = 1;                                       \
+        if ((fp)->value_buffer_used + (l) <= sizeof((fp)->value_buffer)) {        \
+            memcpy(fp->value_buffer + (fp)->value_buffer_used, (v), (l));     \
+            (fp)->fields[(field_id)].value = (fp)->value_buffer + (fp)->value_buffer_used;  \
+            (fp)->fields[(field_id)].value_len = (l);                             \
+            (fp)->value_buffer_used += (l);                                       \
+        } else {                                                                    \
+            (fp)->fields[(field_id)].value = NULL;                                  \
+            (fp)->fields[(field_id)].value_len = 0;                                 \
+        }                                                                           \
+    } while (0)
+
+#define OSFP_FP_INIT_FIELD(fp, field_id) do {                                       \
+        (fp)->fields[(field_id)].name = osfp_fingerprint_get_field_name((field_id));  \
+        (fp)->fields[(field_id)].enabled = 0;                                       \
+        (fp)->fields[(field_id)].value = NULL;                                      \
+        (fp)->fields[(field_id)].value_len = 0;                                     \
+    } while (0)
+
 struct osfp_tcp_opt {
     unsigned char type;
     unsigned char len;
@@ -44,40 +72,6 @@ struct osfp_fingerprint_field fp_fields[OSFP_FIELD_MAX] = {
     {OSFP_FINGERPRINT_FIELD_NAME_OS,                      0, OSFP_FIELD_TYPE_STRING,  0,      NULL,    0},
 };
 
-
-static char *osfp_fingerprint_tcp_options_to_ordered(char *tcp_options, unsigned int len)
-{
-    int i;
-    char *tcp_options_ordered;
-    unsigned tcp_options_ordered_offset;
-    unsigned tcp_options_offset;
-
-    if (tcp_options == NULL && len == 0) {
-        goto exit;
-    }
-
-    tcp_options_ordered = malloc(len + 1);
-    if (tcp_options_ordered == NULL) {
-        goto exit;
-    }
-
-    tcp_options_offset = 0;
-    tcp_options_ordered_offset = 0;
-    while(tcp_options_offset < len) {
-        if (isalpha(tcp_options[tcp_options_offset])) {
-            tcp_options_ordered[tcp_options_ordered_offset] = tcp_options[tcp_options_offset];
-            tcp_options_ordered_offset++;
-        }
-        tcp_options_offset++;
-    }
-
-    tcp_options_ordered[tcp_options_ordered_offset] = 0;
-
-    return tcp_options_ordered;
-exit:
-    return NULL;
-}
-
 static char option_to_ascii(unsigned char type)
 {
     switch (type) {
@@ -122,30 +116,6 @@ static unsigned int compute_ip_ttl(unsigned int ip_ttl)
     return ip_ttl;
 }
 
-static void osfp_fingerprint_setup_field(struct osfp_fingerprint *fp, enum osfp_field_id field_id, void *value, unsigned int len)
-{
-    fp->fields[field_id].name = osfp_fingerprint_get_field_name(field_id);
-    fp->fields[field_id].enabled = 1;
-
-    if (fp->value_buffer_used + len <= sizeof(fp->value_buffer)) {
-        memcpy(fp->value_buffer + fp->value_buffer_used, value, len);
-        fp->fields[field_id].value = fp->value_buffer + fp->value_buffer_used;
-        fp->fields[field_id].value_len = len;
-        fp->value_buffer_used += len;
-    } else {
-        fp->fields[field_id].value = NULL;
-        fp->fields[field_id].value_len = 0;
-    }
-}
-
-static void osfp_fingerprint_init_field(struct osfp_fingerprint *fp, enum osfp_field_id field_id)
-{
-    fp->fields[field_id].name = osfp_fingerprint_get_field_name(field_id);
-    fp->fields[field_id].enabled = 0;
-    fp->fields[field_id].value = NULL;
-    fp->fields[field_id].value_len = 0;
-}
-
 static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned int max_opt_cnt, unsigned char *data, unsigned int len)
 {
     unsigned int offset = 0;
@@ -183,184 +153,37 @@ static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned i
     return tcp_opt_cnt;
 }
 
-static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int opt_len, struct osfp_fingerprint *fp)
+static char *osfp_fingerprint_tcp_options_to_ordered(char *tcp_options, unsigned int len)
 {
-    int ret,i;
+    int i;
+    char *tcp_options_ordered;
+    unsigned tcp_options_ordered_offset;
+    unsigned tcp_options_offset;
 
-    unsigned int tcp_mss;
+    if (tcp_options == NULL && len == 0) {
-    unsigned int tcp_ws;
-    unsigned int tcp_ter;
-    unsigned int tcp_opt_cnt;
-    struct osfp_tcp_opt tcp_opts[OSFP_TCP_OPTMAX];
-
-    char options[OSFP_TCP_OPTLENMAX];
-    char options_ordered[OSFP_TCP_OPTLENMAX];
-    unsigned int offset = 0;
-    unsigned int maxoffset = sizeof(options) - 3; //for shortest "E,"
-    unsigned int ordered_offset = 0;
-    unsigned int ordered_maxoffset = sizeof(options_ordered) - 1;
-
-    if (opt_data == NULL || opt_len == 0 || fp == NULL) {
         goto exit;
     }
 
-    tcp_opt_cnt = decode_tcp_options(tcp_opts, OSFP_TCP_OPTMAX, opt_data, opt_len);
+    tcp_options_ordered = malloc(len + 1);
-
+    if (tcp_options_ordered == NULL) {
-    for (i = 0; i < tcp_opt_cnt && offset < maxoffset && ordered_offset < ordered_maxoffset; i++) {
+        goto exit;
-        struct osfp_tcp_opt *opt = &tcp_opts[i];
-
-        char letter = option_to_ascii(opt->type);
-        options[offset++] = letter;
-        options_ordered[ordered_offset++] = letter;
-
-        switch (opt->type) {
-            case OSFP_TCP_OPT_EOL:
-            case OSFP_TCP_OPT_NOP:
-                break;
-            case OSFP_TCP_OPT_MSS:
-                if (opt->len != OSFP_TCP_OPT_MSS_LEN) {
-                    break;
-                }
-                tcp_mss = (unsigned int)ntohs(*(unsigned short *)opt->data);
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_MSS, &tcp_mss, sizeof(tcp_mss));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_mss);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_WSCALE:
-                if (opt->len != OSFP_TCP_OPT_WS_LEN) {
-                    break;
-                }
-                tcp_ws = (unsigned int)*(unsigned char *)opt->data;
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_WINDOW_SCALING, &tcp_ws, sizeof(tcp_ws));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ws);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_TIMESTAMP:
-                if (opt->len != OSFP_TCP_OPT_TS_LEN) {
-                    break;
-                }
-                tcp_ter = ntohl(*(unsigned int *)(opt->data + 4));
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY, &tcp_ter, sizeof(tcp_ter));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ter);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_SACKOK:
-            case OSFP_TCP_OPT_SACK:
-            case OSFP_TCP_OPT_ECHO:
-            case OSFP_TCP_OPT_ECHOREPLY:
-            case OSFP_TCP_OPT_POCONN:
-            case OSFP_TCP_OPT_POSVC:
-                break;
-            default:
-                ret = snprintf(options + offset, sizeof(options), "%u", opt->type);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
     }
 
-        options[offset++] = ',';
+    tcp_options_offset = 0;
-        options[offset] = 0;
+    tcp_options_ordered_offset = 0;
-        options_ordered[ordered_offset] = 0;
+    while(tcp_options_offset < len) {
+        if (isalpha(tcp_options[tcp_options_offset])) {
+            tcp_options_ordered[tcp_options_ordered_offset] = tcp_options[tcp_options_offset];
+            tcp_options_ordered_offset++;
+        }
+        tcp_options_offset++;
     }
 
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OPTIONS, options, strlen(options) + 1);
+    tcp_options_ordered[tcp_options_ordered_offset] = 0;
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED, options_ordered, strlen(options_ordered) + 1);
 
-    return 0;
+    return tcp_options_ordered;
 exit:
-    return -1;
+    return NULL;
-}
-
-static int osfp_fingerprinting_tcp(struct tcphdr *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp)
-{
-    unsigned int tcp_off;
-    unsigned int tcp_window_size;
-    unsigned int tcp_flags;
-
-    if (tcph == NULL || tcph_len > OSFP_TCP_DATA_OFF_MAX || fp == NULL) {
-        goto exit;
-    }
-
-    tcp_off = tcph->doff << 2;
-    tcp_window_size = ntohs(tcph->window);
-    tcp_flags = *((unsigned char *)&tcph->ack_seq + 5);
-
-    if (tcp_off != tcph_len) {
-        goto exit;
-    }
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OFF, &tcp_off, sizeof(tcp_off));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_WINDOW_SIZE, &tcp_window_size, sizeof(tcp_window_size));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_FLAGS, &tcp_flags, sizeof(tcp_flags));
-
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_TIMESTAMP);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_WINDOW_SCALING);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_MSS);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_OPTIONS);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED);
-
-    // tcp options
-    if (tcp_off > OSFP_TCP_HEADER_LEN) {
-        osfp_fingerprinting_tcp_option((unsigned char *)tcph + OSFP_TCP_HEADER_LEN, tcp_off - OSFP_TCP_HEADER_LEN, fp);
-    }
-
-    return 0;
-exit:
-    return -1;
-}
-
-static int osfp_fingerprinting_ipv4(struct iphdr *iph, struct osfp_fingerprint *fp)
-{
-    if (iph == NULL || fp == NULL) {
-        goto exit;
-    }
-
-    unsigned int ip_id = !!iph->id;
-    unsigned int ip_tos = iph->tos;
-    unsigned int ip_total_length = ntohs(iph->tot_len);
-    unsigned int ip_ttl = compute_ip_ttl(iph->ttl);
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_ID, &ip_id, sizeof(ip_id));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOS, &ip_tos, sizeof(ip_tos));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
-
-    return 0;
-exit:
-    return -1;
-}
-
-static int osfp_fingerprinting_ipv6(struct ip6_hdr *iph, struct osfp_fingerprint *fp)
-{
-    if (iph == NULL || fp == NULL) {
-        goto exit;
-    }
-
-    //unsigned int ip_id = 0;
-    //unsigned int ip_tos = 0;
-    unsigned int ip_total_length = OSFP_IPV6_HEADER_LEN + ntohs(iph->ip6_ctlun.ip6_un1.ip6_un1_plen);
-    unsigned int ip_ttl = compute_ip_ttl(iph->ip6_ctlun.ip6_un1.ip6_un1_hlim);
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
-
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_IP_ID);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_IP_TOS);
-
-    return 0;
-exit:
-    return -1;
 }
 
 int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format)
@@ -405,6 +228,75 @@ int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsi
     return strlen(strbuf) + 1;
 }
 
+struct osfp_fingerprint *osfp_fingerprint_from_cjson(void *cjson_obj)
+{
+    int i;
+    cJSON *root;
+    cJSON *field;
+    void *value_ptr;
+    unsigned int value_len;
+    struct osfp_fingerprint *fp;
+
+    root = (cJSON *)cjson_obj;
+
+    if (root == NULL) {
+        goto exit;
+    }
+
+    fp = calloc(1, sizeof(struct osfp_fingerprint));
+    if (fp == NULL) {
+        goto exit;
+    }
+
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
+        field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(i));
+        if (field == NULL) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
+        switch (field->type) {
+            case cJSON_Number:
+                value_ptr = (void *)&field->valueint;
+                value_len = sizeof(field->valueint);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
+                break;
+            case cJSON_String:
+                value_ptr = (void *)field->valuestring;
+                value_len = strlen(field->valuestring) + 1;
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                break;
+            case cJSON_NULL:
+                //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
+                break;
+            default:
+                goto exit;
+        }
+    }
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_OS);
+
+    return fp;
+exit:
+    if (fp) {
+        free(fp);
+    }
+    return NULL;
+}
+
 int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
 {
     int ret, i;
@@ -440,7 +332,8 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
     }
 
     for (i = 0; i < OSFP_FIELD_OS; i++) {
-        if (!fp_fields[i].enabled) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
             continue;
         }
 
@@ -453,15 +346,23 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
             case cJSON_Number:
                 value_ptr = (void *)&field->valueint;
                 value_len = sizeof(field->valueint);
-                osfp_fingerprint_setup_field(fp, i, value_ptr, value_len);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
                 break;
             case cJSON_String:
                 value_ptr = (void *)field->valuestring;
                 value_len = strlen(field->valuestring) + 1;
-                osfp_fingerprint_setup_field(fp, i, value_ptr, value_len);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
                 break;
             case cJSON_NULL:
-                //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
+                //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
                 break;
             default:
                 goto exit;
@@ -476,11 +377,194 @@ exit:
     return ret;
 }
 
+static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int opt_len, struct osfp_fingerprint *fp)
+{
+    int ret,i;
+
+    unsigned int tcp_mss;
+    unsigned int tcp_ws;
+    unsigned int tcp_ts;
+    unsigned int tcp_ter;
+    unsigned int tcp_opt_cnt;
+    struct osfp_tcp_opt tcp_opts[OSFP_TCP_OPTMAX];
+
+    char options[OSFP_TCP_OPTLENMAX];
+    char options_ordered[OSFP_TCP_OPTLENMAX];
+    unsigned int offset = 0;
+    unsigned int maxoffset = sizeof(options) - 3; //for shortest "E,"
+    unsigned int ordered_offset = 0;
+    unsigned int ordered_maxoffset = sizeof(options_ordered) - 1;
+
+    if (opt_data == NULL || opt_len == 0 || fp == NULL) {
+        goto exit;
+    }
+
+    tcp_opt_cnt = decode_tcp_options(tcp_opts, OSFP_TCP_OPTMAX, opt_data, opt_len);
+
+    for (i = 0; i < tcp_opt_cnt && offset < maxoffset && ordered_offset < ordered_maxoffset; i++) {
+        struct osfp_tcp_opt *opt = &tcp_opts[i];
+
+        char letter = option_to_ascii(opt->type);
+        options[offset++] = letter;
+        options_ordered[ordered_offset++] = letter;
+
+        switch (opt->type) {
+            case OSFP_TCP_OPT_EOL:
+            case OSFP_TCP_OPT_NOP:
+                break;
+            case OSFP_TCP_OPT_MSS:
+                if (opt->len != OSFP_TCP_OPT_MSS_LEN) {
+                    break;
+                }
+                tcp_mss = (unsigned int)ntohs(*(unsigned short *)opt->data);
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_MSS, &tcp_mss, sizeof(tcp_mss));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_mss);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_WSCALE:
+                if (opt->len != OSFP_TCP_OPT_WS_LEN) {
+                    break;
+                }
+                tcp_ws = (unsigned int)*(unsigned char *)opt->data;
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SCALING, &tcp_ws, sizeof(tcp_ws));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ws);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_TIMESTAMP:
+                if (opt->len != OSFP_TCP_OPT_TS_LEN) {
+                    break;
+                }
+                tcp_ts = !!ntohl(*(unsigned int *)(opt->data));
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP, &tcp_ts, sizeof(tcp_ts));
+                tcp_ter = !!ntohl(*(unsigned int *)(opt->data + 4));
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY, &tcp_ter, sizeof(tcp_ter));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ter);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_SACKOK:
+            case OSFP_TCP_OPT_SACK:
+            case OSFP_TCP_OPT_ECHO:
+            case OSFP_TCP_OPT_ECHOREPLY:
+            case OSFP_TCP_OPT_POCONN:
+            case OSFP_TCP_OPT_POSVC:
+                break;
+            default:
+                ret = snprintf(options + offset, sizeof(options), "%u", opt->type);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+        }
+
+        options[offset++] = ',';
+        options[offset] = 0;
+        options_ordered[ordered_offset] = 0;
+    }
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OPTIONS, options, strlen(options) + 1);
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED, options_ordered, strlen(options_ordered) + 1);
+
+    return 0;
+exit:
+    return -1;
+}
+
+static int osfp_fingerprinting_tcp(struct tcphdr *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp)
+{
+    unsigned int tcp_off;
+    unsigned int tcp_window_size;
+    unsigned int tcp_flags;
+
+    if (tcph == NULL || tcph_len > OSFP_TCP_DATA_OFF_MAX || fp == NULL) {
+        goto exit;
+    }
+
+    tcp_off = tcph->doff << 2;
+    tcp_window_size = ntohs(tcph->window);
+    tcp_flags = *((unsigned char *)&tcph->ack_seq + 5);
+
+    if (tcp_off != tcph_len) {
+        goto exit;
+    }
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OFF, &tcp_off, sizeof(tcp_off));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SIZE, &tcp_window_size, sizeof(tcp_window_size));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_FLAGS, &tcp_flags, sizeof(tcp_flags));
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SCALING);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_MSS);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_OPTIONS);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED);
+
+    // tcp options
+    if (tcp_off > OSFP_TCP_HEADER_LEN) {
+        osfp_fingerprinting_tcp_option((unsigned char *)tcph + OSFP_TCP_HEADER_LEN, tcp_off - OSFP_TCP_HEADER_LEN, fp);
+    }
+
+    return 0;
+exit:
+    return -1;
+}
+
+static int osfp_fingerprinting_ipv4(struct iphdr *iph, struct osfp_fingerprint *fp)
+{
+    if (iph == NULL || fp == NULL) {
+        goto exit;
+    }
+
+    unsigned int ip_id = !!iph->id;
+    unsigned int ip_tos = iph->tos;
+    unsigned int ip_total_length = ntohs(iph->tot_len);
+    unsigned int ip_ttl = compute_ip_ttl(iph->ttl);
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_ID, &ip_id, sizeof(ip_id));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOS, &ip_tos, sizeof(ip_tos));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
+
+    return 0;
+exit:
+    return -1;
+}
+
+static int osfp_fingerprinting_ipv6(struct ip6_hdr *iph, struct osfp_fingerprint *fp)
+{
+    if (iph == NULL || fp == NULL) {
+        goto exit;
+    }
+
+    //unsigned int ip_id = 0;
+    //unsigned int ip_tos = 0;
+    unsigned int ip_total_length = OSFP_IPV6_HEADER_LEN + ntohs(iph->ip6_ctlun.ip6_un1.ip6_un1_plen);
+    unsigned int ip_ttl = compute_ip_ttl(iph->ip6_ctlun.ip6_un1.ip6_un1_hlim);
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_IP_ID);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_IP_TOS);
+
+    return 0;
+exit:
+    return -1;
+}
+
 int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version)
 {
     int ret = OSFP_EINVAL;
 
-    if (iph == NULL || tcph == NULL || fp == NULL) {
+    if (iph == NULL || tcph == NULL || tcph_len == 0 || fp == NULL) {
         goto exit;
     }
 
@@ -507,7 +591,7 @@ int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tc
         goto exit;
     }
 
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_OS, OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME, strlen(OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME) + 1);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_OS);
 
     return 0;
 exit:
@@ -531,7 +615,7 @@ int test_osfp_fingerprinting_ipv4(void)
     };
 
     char str_buf[2048] = "";
-    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":\"OSFP_UNKNOWN\"}";
+    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":null}";
     struct osfp_fingerprint fp = {0};
 
     ret = osfp_fingerprinting(iph, tcph, 32, &fp, 4);
@@ -571,7 +655,7 @@ int test_osfp_fingerprinting_ipv6(void)
     };
 
     char str_buf[2048] = "";
-    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":\"OSFP_UNKNOWN\"}";
+    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":null}";
     struct osfp_fingerprint fp = {0};
 
     ret = osfp_fingerprinting(iph, tcph, 32, &fp, 4);

src/osfp_fingerprint.h

@@ -65,8 +65,10 @@ static inline unsigned int osfp_fingerprint_get_field_type(enum osfp_field_id fi
     return fp_fields[field_id].type;
 }
 
+struct osfp_fingerprint *osfp_fingerprint_from_cjson(void *root);
 int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str);
 int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format);
+
 int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version);
 
 int test_osfp_fingerprinting_ipv4(void);

src/osfp_log.c

@@ -1,3 +1,8 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <stdarg.h>
+
 #include "osfp_common.h"
 #include "osfp_log.h"
 

src/osfp_score_db.c

@@ -1,3 +1,8 @@
+#include <stdio.h>
+#include <sys/param.h>
+
+#include "cJSON.h"
+
 #include "osfp_common.h"
 
 #include "osfp.h"
@@ -5,20 +10,29 @@
 #include "osfp_score_db.h"
 #include "osfp_log.h"
 
-#define PERFECT_SCORE_EXPECTED_RATE 0.5f
+/*
+ * 一个字段的值，最多能命中单个操作系统的，指纹总数的百分比
+ * 例如：linux 总指纹数量 100，其中 ip_id 字段值为 1 的指纹数为80，那么 ip_id
+ * 字段的得分最大为 FIELD_VALUE_DUP_RATE_MAX * 100 = 50
+ */
 #define FIELD_VALUE_DUP_RATE_MAX 0.5f
 
+/*
+ * 由于 FIELD_VALUE_DUP_RATE_MAX 设置了单个字段得分的最大百分比，所以在归一化时，应除以这个百分比
+ */
+#define PERFECT_SCORE_EXPECTED_RATE (FIELD_VALUE_DUP_RATE_MAX)
+
 #define OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX 65536
 
 struct osfp_score_db_array_data {
-    struct osfp_os_class_score *array_head;
+    struct osfp_field_value_count *array_head;
     unsigned int    array_len;
 };
 
 struct osfp_score_db_hash_element {
     char *key;
     unsigned int keylen;
-    struct osfp_os_class_score *score;
+    struct osfp_field_value_count *fvc;
     UT_hash_handle hh;
 };
 
@@ -26,13 +40,13 @@ struct osfp_score_db_hash_data {
     struct osfp_score_db_hash_element  *hash_head;
 };
 
-static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len)
+static int osfp_score_db_array_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len)
 {
     int ret = -1, i;
     unsigned int index;
     struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data;
 
-    if (array_data == NULL || os_class_score == NULL || value == NULL || len != sizeof(unsigned int)) {
+    if (array_data == NULL || fvc == NULL || value == NULL || len != sizeof(unsigned int)) {
         goto exit;
     }
 
@@ -47,7 +61,7 @@ static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_cl
     }
 
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        array_data->array_head[index].scores[i] += os_class_score->scores[i];
+        array_data->array_head[index].counts[i] += fvc->counts[i];
     }
 
     return 0;
@@ -55,7 +69,7 @@ exit:
     return ret;
 }
 
-static struct osfp_os_class_score *osfp_score_db_array_match(void *data, void *value, unsigned int len)
+static struct osfp_field_value_count *osfp_score_db_array_match(void *data, void *value, unsigned int len)
 {
     unsigned int index;
     struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data;
@@ -84,7 +98,7 @@ static void *osfp_score_db_array_create(void)
         return NULL;
     }
 
-    array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_os_class_score));
+    array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_field_value_count));
     if (array_data->array_head == NULL) {
         free(array_data);
         return NULL;
@@ -106,13 +120,13 @@ static void osfp_score_db_array_destroy(void *data) {
     }
 }
 
-static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len)
+static int osfp_score_db_hash_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len)
 {
     int ret = -1, i;
     struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data;
     struct osfp_score_db_hash_element *element = NULL;
 
-    if (hash_data == NULL || os_class_score == NULL || value == NULL || len == 0) {
+    if (hash_data == NULL || fvc == NULL || value == NULL || len == 0) {
         goto exit;
     }
 
@@ -124,8 +138,8 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla
         }
         element->key = strdup(value);
         element->keylen = len;
-        element->score = (struct osfp_os_class_score *)calloc(1, sizeof(struct osfp_os_class_score));
+        element->fvc = (struct osfp_field_value_count *)calloc(1, sizeof(struct osfp_field_value_count));
-        if (element->score == NULL) {
+        if (element->fvc == NULL) {
             free(element);
             element = NULL;
             goto exit;
@@ -134,7 +148,7 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla
     }
 
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        element->score->scores[i] += os_class_score->scores[i];
+        element->fvc->counts[i] += fvc->counts[i];
     }
 
     return 0;
@@ -142,7 +156,7 @@ exit:
     return ret;
 }
 
-static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *value, unsigned int len)
+static struct osfp_field_value_count *osfp_score_db_hash_match(void *data, void *value, unsigned int len)
 {
     struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data;
     struct osfp_score_db_hash_element *element = NULL;
@@ -160,7 +174,7 @@ static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *va
         return NULL;
     }
 
-    return element->score;
+    return element->fvc;
 }
 
 static void *osfp_score_db_hash_create(void)
@@ -181,8 +195,8 @@ static void osfp_score_db_hash_destroy(void *data) {
                     if (element->key) {
                         free(element->key);
                     }
-                    if (element->score) {
+                    if (element->fvc) {
-                        free(element->score);
+                        free(element->fvc);
                     }
                     free(element);
                 }
@@ -192,10 +206,136 @@ static void osfp_score_db_hash_destroy(void *data) {
     }
 }
 
+void osfp_score_db_debug_print(struct osfp_score_db *score_db)
+{
+    int i;
+    printf("score_db:\n");
+    printf("entry_count: %u\n", score_db->entry_count);
+    printf("total_weight: %u\n", score_db->total_weight);
+
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        const char *name = osfp_os_class_id_to_name(i);
+        printf("os class %s ", name);
+
+        printf("entry_count: %u\n", score_db->os_entry_count[i]);
+        printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_entry_count[i]);
+    }
+
+    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+        printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled);
+        printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type);
+        printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count);
+    }
+}
+
+const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
+{
+    int ret, i;
+    unsigned int value_buffer_used = 0;
+    char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX];
+    struct osfp_prefilter_hash_element *element = NULL;
+
+    if (score_db->prefilter_head == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            continue;
+        }
+        if (fp->fields[i].value && fp->fields[i].value_len != 0) {
+            memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len);
+            value_buffer_used += fp->fields[i].value_len;
+        }
+    }
+
+    HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element);
+    if (element == NULL) {
+        return NULL;
+    }
+
+    if (element->repeated) {
+        return NULL;
+    }
+
+    memset(result_score, 0, sizeof(struct osfp_os_class_score));
+    result_score->scores[element->os_class] = OSFP_PERCENTILE;
+
+    return (const char *)element->fp_json;
+}
+
+int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
+{
+    int ret = -1, i, j;
+
+    unsigned int coefficient;
+    unsigned int matched_count;
+    unsigned int os_entry_count;
+    unsigned int total_weight;
+    unsigned int field_weight;
+
+    struct osfp_fingerprint_field *field;
+    struct osfp_field_value_count *fvc; // field_value_count
+    struct osfp_field_score_db *fdb;
+
+    if (score_db == NULL || fp == NULL || result_score == NULL) {
+        goto exit;
+    }
+
+    // score
+    memset(result_score, 0, sizeof(struct osfp_os_class_score));
+
+    total_weight = score_db->total_weight;
+    if (total_weight == 0) {
+        goto exit;
+    }
+
+    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+        fdb = &score_db->field_score_dbs[i];
+        if (!fdb->enabled) {
+            continue;
+        }
+
+        field = &fp->fields[i];
+        if (!field->enabled) {
+            continue;
+        }
+
+        fvc = fdb->match(fdb->data, field->value, field->value_len);
+        if (fvc == NULL) {
+            continue;
+        }
+
+        coefficient = fdb->coefficient;
+
+        for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
+            os_entry_count = score_db->os_entry_count[j];
+            matched_count = MIN(fvc->counts[j], os_entry_count * FIELD_VALUE_DUP_RATE_MAX);
+
+            if (os_entry_count == 0 || matched_count == 0) {
+                continue;
+            }
+
+            if (0 == flags || flags & OSFP_BIT_U32(j)) {
+                result_score->scores[j] += coefficient * matched_count / os_entry_count;
+            }
+        }
+
+        // if tcp options matched tcp options ordered is not needed
+        if (i == OSFP_FIELD_TCP_OPTIONS) {
+            i++;
+        }
+    }
+
+    return 0;
+exit:
+    return ret;
+}
+
 static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field, enum osfp_os_class_id os_class)
 {
     int ret = -1;
-    struct osfp_os_class_score os_class_score = {0};
+    struct osfp_field_value_count field_value_count = {0};
 
     void *value_ptr;
     unsigned int value_len;
@@ -216,9 +356,9 @@ static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field
             goto exit;
     }
 
-    os_class_score.scores[os_class] = 1;
+    field_value_count.counts[os_class] = 1;
 
-    ret = db->add(db->data, &os_class_score, value_ptr, value_len);
+    ret = db->add(db->data, &field_value_count, value_ptr, value_len);
     if (ret != 0) {
         goto exit;
     }
@@ -241,6 +381,7 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
         goto exit;
     }
 
+    // get os class
     field = cJSON_GetObjectItem(entry, osfp_fingerprint_get_field_name(OSFP_FIELD_OS));
     if (field == NULL || field->valuestring == NULL) {
         goto exit;
@@ -251,6 +392,31 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
         goto exit;
     }
 
+    // prefileter
+    struct osfp_prefilter_hash_element *element = NULL;
+    struct osfp_fingerprint *fp = osfp_fingerprint_from_cjson(entry);
+    if (fp == NULL) {
+        goto exit;
+    }
+    HASH_FIND(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    if (element == NULL) {
+        element = (struct osfp_prefilter_hash_element *)calloc(1, sizeof(struct osfp_prefilter_hash_element));
+        if (element == NULL) {
+            free(fp);
+            goto exit;
+        }
+
+        element->fp_json = cJSON_Print(entry);
+        element->fp = fp;
+        element->os_class = os_class;
+        HASH_ADD_KEYPTR(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    } else {
+        // TODO: same fingerprints with different os should not insert into prefilter hash table, now just tag
+        element->repeated++;
+        free(fp);
+    }
+
+    // field score db
     for (i = 0; i < OSFP_FIELD_OS; i++) {
         db = &score_db->field_score_dbs[i];
         if (db == NULL) {
@@ -277,210 +443,58 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
     }
 
     score_db->entry_count++;
-    score_db->os_class_entry_count[os_class]++;
+    score_db->os_entry_count[os_class]++;
 
     return 0;
 exit:
     return ret;
 }
 
-char *osfp_score_db_read_file(char *fp_file)
-{
-    int ret = -1;
-    char *file_buffer = NULL;
-    unsigned int file_len = 0;
-    FILE *fp = NULL;
-    struct stat st;
-
-    if (0 > stat(fp_file, &st)) {
-        osfp_log_error("stat() failed on '%s'.\n", fp_file);
-        goto exit;
-    }
-
-    if (st.st_size == 0) { 
-        osfp_log_error("Empty file: '%s'.\n", fp_file);
-        goto exit;
-    }
-
-    file_len = (unsigned int)st.st_size;
-    file_buffer = malloc(file_len + 1);
-    if (file_buffer == NULL) {
-        osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file);
-        goto exit;
-    }
-
-    fp = fopen(fp_file, "r");
-    if (fp == NULL) {
-        osfp_log_error("Cannot open '%s' for reading.\n", fp_file);
-        goto exit;
-    }
-
-    ret = fread(file_buffer, 1, file_len, fp);
-    if (ret < 0) {
-        osfp_log_error("fread() failed on '%s'.\n", fp_file);
-        free(file_buffer);
-        fclose(fp);
-        goto exit;
-    }
-
-    fclose(fp);
-
-    file_buffer[file_len] = 0;
-
-    return file_buffer;
-exit:
-    return NULL;
-}
-
 int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file)
 {
-    int ret = OSFP_EINVAL, i, count;
+    int ret = -1, i, count;
     char *file_buffer;
-    cJSON *root = NULL;
     cJSON *entry;
+    cJSON *root = NULL;
     struct osfp_field_score_db *field_score_db;
 
-    if (score_db == NULL) {
+    if (score_db == NULL || fp_file == NULL) {
         goto exit;
     }
 
-    file_buffer = osfp_score_db_read_file(fp_file);
+    file_buffer = osfp_read_file(fp_file);
     if (file_buffer == NULL) {
         osfp_log_error("read file: '%s'\n", fp_file);
-        ret = OSFP_ERR_SCORE_DB_READ_FILE;
         goto exit;
     }
 
     root = cJSON_Parse(file_buffer);
     if (root == NULL) {
         osfp_log_error("parse json: '%s'\n", fp_file);
-        ret = OSFP_ERR_SCORE_DB_PARSE_FILE;
         goto exit;
     }
 
     count = cJSON_GetArraySize(root);
-
     for (i = 0; i < count; i++) {
         entry = cJSON_GetArrayItem(root, i);
-        if (entry) {
         ret = osfp_score_db_load_entry(score_db, entry);
         if (ret != 0) {
             osfp_log_debug("json entry load failed.\n%s\n", cJSON_Print(entry));
             continue;
         }
     }
-    }
 
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+    ret = 0;
-        field_score_db = &score_db->field_score_dbs[i];
-        if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS_ORDERED) {
-            score_db->perfect_score += osfp_fingerprint_get_field_importance(i);
-        }
-    }
-
-    score_db->perfect_score = score_db->perfect_score * PERFECT_SCORE_EXPECTED_RATE;
-
-    ret = OSFP_NOERR;
 exit:
     if (root) {
         cJSON_Delete(root);
     }
     if (file_buffer) {
-        free((char*)file_buffer);
+        free(file_buffer);
     }
     return ret;
 }
 
-int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
-{
-    int ret = OSFP_EINVAL, i, j;
-
-    unsigned int tmp_score;
-    unsigned int perfect_score;
-    unsigned int entry_count;
-    unsigned int importance;
-
-    struct osfp_os_class_score *os_class_score_matched;
-    enum osfp_os_class_id os_class_id;
-
-    struct osfp_fingerprint_field *field;
-    struct osfp_field_score_db *field_score_db;
-
-    if (score_db == NULL || fp == NULL || result_score == NULL) {
-        goto exit;
-    }
-
-    memset(result_score, 0, sizeof(struct osfp_os_class_score));
-
-    perfect_score = score_db->perfect_score;
-    if (perfect_score == 0) {
-        goto exit;
-    }
-
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
-        field_score_db = &score_db->field_score_dbs[i];
-        if (!field_score_db->enabled) {
-            continue;
-        }
-
-        field = &fp->fields[i];
-        if (!field->enabled) {
-            continue;
-        }
-
-        os_class_score_matched = field_score_db->match(field_score_db->data, field->value, field->value_len);
-        if (os_class_score_matched == NULL) {
-            continue;
-        }
-
-        importance = osfp_fingerprint_get_field_importance(i);
-
-        for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
-            entry_count = score_db->os_class_entry_count[j];
-            tmp_score = os_class_score_matched->scores[j];
-            tmp_score = tmp_score < (entry_count * FIELD_VALUE_DUP_RATE_MAX) ? tmp_score : (entry_count * FIELD_VALUE_DUP_RATE_MAX);
-            if (entry_count == 0 || tmp_score == 0) {
-                continue;
-            }
-            if (0 == flags || flags & OSFP_BIT_U32(j)) {
-                osfp_log_debug("%s %s: ((%d * %u / %u) * %u ) / %u\n", osfp_fingerprint_get_field_name(i), osfp_os_class_id_to_name(j), OSFP_PERCENTILE, importance, perfect_score, os_class_score_matched->scores[j], entry_count);
-                result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * tmp_score) / entry_count;
-            }
-        }
-
-        if (i == OSFP_FIELD_TCP_OPTIONS) {
-            // if OSFP_FIELD_TCP_OPTIONS matched OSFP_FIELD_TCP_OPTIONS_ORDERED is not needed
-            i++;
-        }
-    }
-
-    return OSFP_NOERR;
-exit:
-    return ret;
-}
-
-void osfp_score_db_debug_print(struct osfp_score_db *score_db)
-{
-    int i;
-    printf("score_db:\n");
-    printf("entry_count: %u\n", score_db->entry_count);
-    printf("perfect_score: %u\n", score_db->perfect_score);
-
-    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        const char *name = osfp_os_class_id_to_name(i);
-        printf("os class %s ", name);
-
-        printf("entry_count: %u\n", score_db->os_class_entry_count[i]);
-        printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_class_entry_count[i]);
-    }
-
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
-        printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled);
-        printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type);
-        printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count);
-    }
-}
-
 struct osfp_score_db *osfp_score_db_create(void)
 {
     int i;
@@ -501,6 +515,12 @@ struct osfp_score_db *osfp_score_db_create(void)
             continue;
         }
 
+        db->weight = osfp_fingerprint_get_field_importance(i);
+        // tcp options ordered and tcp options overlap
+        if (i != OSFP_FIELD_TCP_OPTIONS_ORDERED) {
+            score_db->total_weight += db->weight;
+        }
+
         db->type = osfp_fingerprint_get_field_type(i);
         switch (db->type) {
             case OSFP_FIELD_TYPE_UINT:
@@ -516,17 +536,22 @@ struct osfp_score_db *osfp_score_db_create(void)
                 db->match = osfp_score_db_hash_match;
                 break;
             default:
-                osfp_log_debug("fingerprint field unsupported type: %u", db->type);
+                osfp_log_error("fingerprint field unsupported type: %u", db->type);
                 goto exit;
         }
 
         db->data = db->create();
         if (db->data == NULL) {
-            osfp_log_debug("field db create failed. field: %s", osfp_fingerprint_get_field_name(i));
+            osfp_log_error("field db create failed. field: %s", osfp_fingerprint_get_field_name(i));
             goto exit;
         }
     }
 
+    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+        db = &score_db->field_score_dbs[i];
+        db->coefficient = (OSFP_PERCENTILE / PERFECT_SCORE_EXPECTED_RATE) * ((float)db->weight / (float)score_db->total_weight);
+    }
+
     return score_db;
 exit:
     if (score_db) {
@@ -539,8 +564,27 @@ void osfp_score_db_destroy(struct osfp_score_db *score_db)
 {
     int i;
     struct osfp_field_score_db *db;
+    struct osfp_prefilter_hash_element *element = NULL;
+    struct osfp_prefilter_hash_element *tmp = NULL;
 
     if (score_db) {
+        // prefilter
+        if (score_db->prefilter_head) {
+            HASH_ITER(hh, score_db->prefilter_head, element, tmp) {
+                HASH_DELETE(hh, score_db->prefilter_head, element);
+                if (element) {
+                    if (element->fp) {
+                        free(element->fp);
+                    }
+                    if (element->fp_json) {
+                        free(element->fp_json);
+                    }
+                    free(element);
+                }
+            }
+        }
+
+        // field score db
         for (i = 0; i < OSFP_FIELD_MAX; i++) {
             db = &score_db->field_score_dbs[i];
             if (db->destroy && db->data) {
@@ -600,7 +644,7 @@ int test_osfp_score_db(void)
         goto exit;
     }
 
-    if (db->os_class_entry_count[OSFP_OS_CLASS_ANDROID] != 1) {
+    if (db->os_entry_count[OSFP_OS_CLASS_ANDROID] != 1) {
         goto exit;
     }
 

src/osfp_score_db.h

@@ -1,12 +1,14 @@
 #ifndef __OSFP_SCORE_DB_H__
 #define __OSFP_SCORE_DB_H__
 
+#include "uthash.h"
+
 #include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_common.h"
 
-struct osfp_os_class_score {
+struct osfp_field_value_count {
-    unsigned int scores[OSFP_OS_CLASS_MAX];
+    unsigned int counts[OSFP_OS_CLASS_MAX];
 };
 
 struct osfp_field_score_db {
@@ -14,24 +16,42 @@ struct osfp_field_score_db {
     unsigned int    type;
     unsigned int    entry_count;
 
+    unsigned int    weight;
+    float           coefficient;
+
     void            *data;
 
     void *(*create)(void);
     void (*destroy)(void *);
-    int (*add)(void *data, struct osfp_os_class_score *, void *, unsigned int);
+    int (*add)(void *data, struct osfp_field_value_count *, void *, unsigned int);
-    struct osfp_os_class_score *(*match)(void *, void *, unsigned int);
+    struct osfp_field_value_count *(*match)(void *, void *, unsigned int);
+};
+
+struct osfp_prefilter_hash_element {
+    unsigned int os_class;
+    unsigned int repeated;
+
+    struct osfp_fingerprint *fp;
+    char                    *fp_json;
+
+    UT_hash_handle hh;
 };
 
 struct osfp_score_db {
+    struct osfp_prefilter_hash_element *prefilter_head;
+
     unsigned int entry_count;
-    unsigned int perfect_score;
+    unsigned int os_entry_count[OSFP_OS_CLASS_MAX];
-    unsigned int os_class_entry_count[OSFP_OS_CLASS_MAX];
+
+    unsigned int total_weight;
+
     struct osfp_field_score_db field_score_dbs[OSFP_FIELD_MAX];
 };
 
-char *osfp_score_db_read_file(char *fp_file);
 void osfp_score_db_debug_print(struct osfp_score_db *score_db);
 
+const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
+
 int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file);
 int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
 

src/version.map

@@ -1,4 +1,12 @@
 {
-  global: osfp*;GIT_VERSION_*;
+  global:
+    osfp_db_new;
+    osfp_db_free;
+    osfp_ip_identify;
+    osfp_ipv4_identify;
+    osfp_ipv6_identify;
+    osfp_result_os_name_get;
+    osfp_result_score_detail_export;
+    osfp_result_free;
   local: *;
 };

test/test.c

@@ -3,6 +3,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <time.h>
+#include <assert.h>
 
 #include "cJSON.h"
 
@@ -126,13 +127,47 @@ static void print_confusion_matrix(unsigned int *result, unsigned int os_class_m
     printf("miss rate: %u%%\n", 100 * missed / (matched + missed));
 }
 
+struct osfp_result *test_osfp_json_identify(struct osfp_db *db, const char *json_str)
+{
+    int ret = OSFP_EINVAL;
+    struct osfp_fingerprint fp;
+    struct osfp_os_class_score os_class_score;
+    struct osfp_result *result;
+    const char *matched;
+
+    if (db == NULL || json_str == NULL) {
+        goto exit;
+    }
+
+    ret = osfp_fingerprint_from_json(&fp, (char *)json_str);
+    if (ret != 0) {
+        goto exit;
+    }
+
+    matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
+    if (matched == NULL) {
+        ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+        if (ret != 0) {
+            goto exit;
+        }
+    }
+
+    result = osfp_result_build(&os_class_score, matched);
+    if (result == NULL) {
+        goto exit;
+    }
+
+    return result;
+exit:
+    return NULL;
+}
 
 void test_data_prepare()
 {
     char *file_buffer;
 
     if (test_file_path == NULL) {
-        file_buffer = osfp_score_db_read_file(data_file_path);
+        file_buffer = osfp_read_file(data_file_path);
         if (file_buffer == NULL) {
             osfp_log_error("read file: '%s'\n", data_file_path);
             exit(1);
@@ -181,12 +216,14 @@ void test_miss_rate()
     unsigned int other_count = 0;
     unsigned int unknown_count = 0;
     unsigned int identify_failed_count = 0;
+    unsigned int prefiltered_count = 0;
+    unsigned int prefiltered_wrong_count = 0;
     unsigned int wrong_count = 0;
     unsigned int verified_count = 0;
     unsigned int fingerprint_count = 0;
     char *file_buffer;
 
-    file_buffer = osfp_score_db_read_file(test_file_path);
+    file_buffer = osfp_read_file(test_file_path);
     if (file_buffer == NULL) {
         osfp_log_error("read file: '%s'\n", test_file_path);
         exit(1);
@@ -221,7 +258,13 @@ void test_miss_rate()
 
             const char *fp_str = cJSON_PrintUnformatted(entry);
 
-            struct osfp_result *result = osfp_json_identify(osfp_db, fp_str);
+            struct osfp_fingerprint fp = {0};
+            osfp_fingerprint_from_json(&fp, (char*)fp_str);
+            char str_buf[2048] = "";
+            osfp_fingerprint_to_json_buf(&fp, str_buf, 2048, 0);
+            fprintf(log_file_ptr, "%s\n", str_buf);
+
+            struct osfp_result *result = test_osfp_json_identify(osfp_db, fp_str);
             if (result == NULL) {
                 identify_failed_count++;
                 continue;
@@ -229,12 +272,19 @@ void test_miss_rate()
 
             testresult[result->likely_os_class][os_class]++;
 
+            if (result->details[result->likely_os_class].score == 100) {
+                prefiltered_count++;
+            }
+
             if (os_class == result->likely_os_class) {
                 verified_count++;
                 osfp_result_free(result);
                 continue;
             }
 
+            if (result->details[result->likely_os_class].score == 100) {
+                prefiltered_wrong_count++;
+            }
             wrong_count++;
 
             if (result->likely_os_class == OSFP_OS_CLASS_OTHERS) {
@@ -245,11 +295,12 @@ void test_miss_rate()
                 unknown_count++;
             }
 
-            fprintf(log_file_ptr, "expect: %s, result: %s\n", os_class_json->valuestring, osfp_result_os_name_get(result));
+            fprintf(log_file_ptr, "expect: %s, result: %s, \n", os_class_json->valuestring, osfp_result_os_name_get(result));
 
             char *result_json = osfp_result_score_detail_export(result);
             if (result_json) {
                 fprintf(log_file_ptr, "%s\n", result_json);
+                free(result_json);
             } else {
                 fprintf(log_file_ptr, "result detail error:%p\n", result);
             }
@@ -258,8 +309,8 @@ void test_miss_rate()
         }
     }
 
-    printf("total %u, failed %u, pass %u, wrong %u, other %u, unknown %u\n",
+    printf("total %u, failed %u, pass %u, prefiltered %u (wrong: %u), wrong %u, other %u, unknown %u\n",
-                fingerprint_count, identify_failed_count, verified_count, wrong_count, other_count, unknown_count);
+                fingerprint_count, identify_failed_count, verified_count, prefiltered_count, prefiltered_wrong_count,  wrong_count, other_count, unknown_count);
 
     //printf("miss rate: %d%%\n", 100 - (verified_count * 100 / fingerprint_count));