code: clean up

src/osfp.c

@@ -5,14 +5,12 @@
 #include "osfp_score_db.h"
 #include "osfp_log.h"
 
-#define OSFP_DEFAULT_RESULT_BUFLEN_MAX  512
 #define OSFP_LOWEST_SCORE_LIMIT 20
 
-static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score)
+static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score, const char *matched)
 {
     int i;
     unsigned int tmp_score;
-    unsigned int sum_score;
     unsigned int likely_score;
     enum osfp_os_class_id likely_os_class;
     struct osfp_result *result;
@@ -24,8 +22,8 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
 
     likely_score = 0;
     likely_os_class = OSFP_OS_CLASS_OTHERS;
-    sum_score = 0;
 
+    // likely os score
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
         tmp_score = os_class_score->scores[i];
 
@@ -34,20 +32,16 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
             likely_os_class = i;
         }
         result->details[i].score = tmp_score;
-        sum_score += tmp_score;
-    }
-
-    if (sum_score) {
-        for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-            result->details[i].possibility = OSFP_PERCENTILE * result->details[i].score / sum_score;
-        }
     }
 
     if (likely_score == OSFP_PERCENTILE) {
+        // prefiltered
         ;
     } else if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
+        // too low to tell os class
         likely_os_class = OSFP_OS_CLASS_OTHERS;
     } else {
+        // when the tied likely scores appear between win/apple-like/unix-like, we throw unknown
         for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
             if (likely_os_class == i) {
                 continue;
@@ -66,7 +60,7 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
     }
 
     result->likely_os_class = likely_os_class;
-
+    result->matched = matched;
     return result;
 exit:
     return NULL;
@@ -91,22 +85,23 @@ const char *osfp_result_os_name_get(struct osfp_result *result)
 char *osfp_result_score_detail_export(struct osfp_result *result)
 {
     int i;
-    char *result_str = NULL;
+    char *result_str;
-    cJSON *root = NULL;
+    cJSON *root;
     cJSON *array;
     cJSON *os_score;
+    cJSON *matched;
 
     osfp_profile_cycle(c1);
     osfp_profile_cycle(c2);
+
     osfp_profile_get_cycle(c1);
 
     if (result == NULL) {
-        goto exit;
+        return NULL;
     }
 
     if (result->json_str != NULL) {
-        result_str = result->json_str;
+        return result->json_str;
-        goto exit;
     }
 
     root = cJSON_CreateObject();
@@ -134,6 +129,13 @@ char *osfp_result_score_detail_export(struct osfp_result *result)
         }
     }
 
+    if (result->matched) {
+        matched = cJSON_AddObjectToObject(root, "matched");
+        if (matched) {
+            cJSON_AddStringToObject(matched, "fingerprint", result->matched);
+        }
+    }
+
     result_str = cJSON_Print(root);
     if (result_str == NULL) {
         goto exit;
@@ -141,15 +143,15 @@ char *osfp_result_score_detail_export(struct osfp_result *result)
 
     result->json_str = result_str;
 
+    osfp_profile_get_cycle(c2);
+    osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1);
+
+    return result_str;
 exit:
     if (root) {
         cJSON_Delete(root);
     }
-    if (result_str) {
+    return NULL;
-        osfp_profile_get_cycle(c2);
-        osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1);
-    }
-    return result_str;
 }
 
 void osfp_result_free(struct osfp_result *result)
@@ -164,15 +166,16 @@ void osfp_result_free(struct osfp_result *result)
 
 struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
 {
-    int ret = OSFP_EINVAL;
+    int ret;
     struct osfp_fingerprint fp;
     struct osfp_os_class_score os_class_score;
     struct osfp_result *result;
+    const char *matched;
 
     osfp_profile_cycle(c1);
     osfp_profile_cycle(c2);
 
-    if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr == 0) {
+    if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr_len == 0) {
         goto exit;
     }
 
@@ -185,10 +188,10 @@ struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr,
     }
 
     osfp_profile_get_cycle(c1);
-    ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
+    matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
     osfp_profile_get_cycle(c2);
     osfp_profile_counter_update(&osfp_profile_prefilter, c2 - c1);
-    if (ret <= 0) {
+    if (matched == NULL) {
         osfp_profile_get_cycle(c1);
         ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
         osfp_profile_get_cycle(c2);
@@ -199,7 +202,7 @@ struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr,
     }
 
     osfp_profile_get_cycle(c1);
-    result = osfp_result_build(&os_class_score);
+    result = osfp_result_build(&os_class_score, matched);
     osfp_profile_get_cycle(c2);
     osfp_profile_counter_update(&osfp_profile_result_build, c2 - c1);
     if (result == NULL) {
@@ -213,26 +216,45 @@ exit:
 
 struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
 {
-    int ret = OSFP_EINVAL;
+    int ret;
     struct osfp_fingerprint fp;
     struct osfp_os_class_score os_class_score;
     struct osfp_result *result;
+    const char *matched;
+
+    osfp_profile_cycle(c1);
+    osfp_profile_cycle(c2);
 
     if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr_len == 0) {
         goto exit;
     }
 
+    osfp_profile_get_cycle(c1);
     ret = osfp_fingerprinting((unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, &fp, 6);
+    osfp_profile_get_cycle(c2);
+    osfp_profile_counter_update(&osfp_profile_fingerprinting, c2 - c1);
     if (ret != 0) {
         goto exit;
     }
 
-    ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+    osfp_profile_get_cycle(c1);
-    if (ret != 0) {
+    matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
-        goto exit;
+    osfp_profile_get_cycle(c2);
+    osfp_profile_counter_update(&osfp_profile_prefilter, c2 - c1);
+    if (matched == NULL) {
+        osfp_profile_get_cycle(c1);
+        ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+        osfp_profile_get_cycle(c2);
+        osfp_profile_counter_update(&osfp_profile_score, c2 - c1);
+        if (ret != 0) {
+            goto exit;
+        }
     }
 
-    result = osfp_result_build(&os_class_score);
+    osfp_profile_get_cycle(c1);
+    result = osfp_result_build(&os_class_score, matched);
+    osfp_profile_get_cycle(c2);
+    osfp_profile_counter_update(&osfp_profile_result_build, c2 - c1);
     if (result == NULL) {
         goto exit;
     }
@@ -248,8 +270,9 @@ struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str)
     struct osfp_fingerprint fp;
     struct osfp_os_class_score os_class_score;
     struct osfp_result *result;
+    const char *matched;
 
-    if (db == NULL) {
+    if (db == NULL || json_str == NULL) {
         goto exit;
     }
 
@@ -258,15 +281,15 @@ struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str)
         goto exit;
     }
 
-    ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
+    matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
-    if (ret <= 0) {
+    if (matched == NULL) {
         ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
         if (ret != 0) {
             goto exit;
         }
     }
 
-    result = osfp_result_build(&os_class_score);
+    result = osfp_result_build(&os_class_score, matched);
     if (result == NULL) {
         goto exit;
     }
@@ -276,25 +299,27 @@ exit:
     return NULL;
 }
 
-struct osfp_db *osfp_db_new(const char *db_json_file)
+struct osfp_db *osfp_db_new(const char *fp_path)
 {
     int ret;
     struct osfp_db *db;
 
+    if (fp_path == NULL) {
+        goto exit;
+    }
+
+    if (0 != access(fp_path, R_OK)) {
+        goto exit;
+    }
+
     db = calloc(1, sizeof(struct osfp_db));
     if (db == NULL) {
         goto exit;
     }
 
-    if (db_json_file != NULL) {
+    db->db_json_path = strdup((const char*)fp_path);
-        if (0 != access(db_json_file, R_OK)) {
+    if (db->db_json_path == NULL) {
-            goto exit;
+        goto exit;
-        }
-
-        db->db_json_path = strdup((const char*)db_json_file);
-        if (db->db_json_path == NULL) {
-            goto exit;
-        }
     }
 
     db->score_db = (void *)osfp_score_db_create();

src/osfp_common.c

@@ -1,6 +1,7 @@
 #include "osfp_common.h"
 
 #include "osfp.h"
+#include "osfp_log.h"
 
 unsigned int osfp_profile_enable;
 
@@ -25,6 +26,21 @@ const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class)
     return osfp_os_class_name[os_class];
 }
 
+enum osfp_os_class_id osfp_os_class_name_to_id(char *name)
+{
+    int i, namelen;
+    const char *os_class_name;
+
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        os_class_name = osfp_os_class_id_to_name(i);
+        if (0 == strncmp(name, os_class_name, strlen(os_class_name))) {
+            return (enum osfp_os_class_id)i;
+        }
+    }
+
+    return OSFP_OS_CLASS_MAX;
+}
+
 void osfp_profile_counter_print(struct osfp_profile_counter *profile, const char *name)
 {
     printf("profile %s: avg: %lu max: %lu min: %lu curr: %lu total: %lu count: %lu\n",
@@ -69,24 +85,51 @@ void osfp_profile_set(unsigned int enabled)
     osfp_profile_enable = enabled;
 }
 
-enum osfp_os_class_id osfp_os_class_name_to_id(char *name)
+char *osfp_read_file(char *fp_file)
 {
-    enum osfp_os_class_id os_class;
+    int ret = -1;
+    char *file_buffer = NULL;
+    unsigned int file_len = 0;
+    FILE *fp = NULL;
+    struct stat st;
 
-    if (0 == strncmp(name, OSFP_OS_CLASS_NAME_WINDOWS, strlen(OSFP_OS_CLASS_NAME_WINDOWS))) {
+    if (0 > stat(fp_file, &st)) {
-        os_class = OSFP_OS_CLASS_WINDOWS;
+        osfp_log_error("stat() failed on '%s'.\n", fp_file);
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_LINUX, strlen(OSFP_OS_CLASS_NAME_LINUX))) {
+        goto exit;
-        os_class = OSFP_OS_CLASS_LINUX;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_MAC_OS, strlen(OSFP_OS_CLASS_NAME_MAC_OS))) {
-        os_class = OSFP_OS_CLASS_MAC_OS;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_IOS, strlen(OSFP_OS_CLASS_NAME_IOS))) {
-        os_class = OSFP_OS_CLASS_IOS;
-    } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_ANDROID, strlen(OSFP_OS_CLASS_NAME_ANDROID))) {
-        os_class = OSFP_OS_CLASS_ANDROID;
-    } else {
-        os_class = OSFP_OS_CLASS_MAX;
     }
 
-    return os_class;
+    if (st.st_size == 0) { 
+        osfp_log_error("Empty file: '%s'.\n", fp_file);
+        goto exit;
+    }
+
+    file_len = (unsigned int)st.st_size;
+    file_buffer = malloc(file_len + 1);
+    if (file_buffer == NULL) {
+        osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file);
+        goto exit;
+    }
+
+    fp = fopen(fp_file, "r");
+    if (fp == NULL) {
+        osfp_log_error("Cannot open '%s' for reading.\n", fp_file);
+        goto exit;
+    }
+
+    ret = fread(file_buffer, 1, file_len, fp);
+    if (ret < 0) {
+        osfp_log_error("fread() failed on '%s'.\n", fp_file);
+        free(file_buffer);
+        fclose(fp);
+        goto exit;
+    }
+
+    fclose(fp);
+
+    file_buffer[file_len] = 0;
+
+    return file_buffer;
+exit:
+    return NULL;
 }
 

src/osfp_common.h

@@ -11,6 +11,7 @@
 #include <sys/fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/param.h>
 
 #include "utarray.h"
 #include "uthash.h"
@@ -21,8 +22,6 @@
 
 #include "cJSON.h"
 
-#include "osfp.h"
-
 static inline unsigned long long osfp_rdtsc(void)
 {
 	union {
@@ -186,6 +185,7 @@ struct osfp_result {
     char *json_str;                              // JSON 字符串
     enum osfp_os_class_id likely_os_class;        // 最可能的操作系统类别
     struct osfp_result_detail details[OSFP_OS_CLASS_MAX];   // 详细结果数组
+    const char *matched;                        // 精确匹配到的指纹特征
 };
 
 /**
@@ -197,7 +197,7 @@ struct osfp_db {
 };
 
 enum osfp_os_class_id osfp_os_class_name_to_id(char *name);
-
 const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class);
+char *osfp_read_file(char *fp_file);
 
 #endif

src/osfp_fingerprint.c

@@ -270,7 +270,7 @@ struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root)
                 OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
                 break;
             case cJSON_NULL:
-                //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
+                //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
                 break;
             default:
                 goto exit;
@@ -352,7 +352,7 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
                 OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
                 break;
             case cJSON_NULL:
-                //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
+                //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
                 break;
             default:
                 goto exit;
@@ -554,7 +554,7 @@ int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tc
 {
     int ret = OSFP_EINVAL;
 
-    if (iph == NULL || tcph == NULL || fp == NULL) {
+    if (iph == NULL || tcph == NULL || tcph_len == 0 || fp == NULL) {
         goto exit;
     }
 

src/osfp_fingerprint.h

@@ -68,6 +68,7 @@ static inline unsigned int osfp_fingerprint_get_field_type(enum osfp_field_id fi
 struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root);
 int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str);
 int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format);
+
 int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version);
 
 int test_osfp_fingerprinting_ipv4(void);

src/osfp_score_db.c

@@ -5,20 +5,29 @@
 #include "osfp_score_db.h"
 #include "osfp_log.h"
 
-#define PERFECT_SCORE_EXPECTED_RATE 0.5f
+/*
+ * 一个字段的值，最多能命中单个操作系统的，指纹总数的百分比
+ * 例如：linux 总指纹数量 100，其中 ip_id 字段值为 1 的指纹数为80，那么 ip_id
+ * 字段的得分最大为 FIELD_VALUE_DUP_RATE_MAX * 100 = 50
+ */
 #define FIELD_VALUE_DUP_RATE_MAX 0.5f
 
+/*
+ * 由于 FIELD_VALUE_DUP_RATE_MAX 设置了单个字段得分的最大百分比，所以在归一化时，应除以这个百分比
+ */
+#define PERFECT_SCORE_EXPECTED_RATE (FIELD_VALUE_DUP_RATE_MAX)
+
 #define OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX 65536
 
 struct osfp_score_db_array_data {
-    struct osfp_os_class_score *array_head;
+    struct osfp_field_value_count *array_head;
     unsigned int    array_len;
 };
 
 struct osfp_score_db_hash_element {
     char *key;
     unsigned int keylen;
-    struct osfp_os_class_score *score;
+    struct osfp_field_value_count *fvc;
     UT_hash_handle hh;
 };
 
@@ -26,13 +35,13 @@ struct osfp_score_db_hash_data {
     struct osfp_score_db_hash_element  *hash_head;
 };
 
-static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len)
+static int osfp_score_db_array_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len)
 {
     int ret = -1, i;
     unsigned int index;
     struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data;
 
-    if (array_data == NULL || os_class_score == NULL || value == NULL || len != sizeof(unsigned int)) {
+    if (array_data == NULL || fvc == NULL || value == NULL || len != sizeof(unsigned int)) {
         goto exit;
     }
 
@@ -47,7 +56,7 @@ static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_cl
     }
 
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        array_data->array_head[index].scores[i] += os_class_score->scores[i];
+        array_data->array_head[index].counts[i] += fvc->counts[i];
     }
 
     return 0;
@@ -55,7 +64,7 @@ exit:
     return ret;
 }
 
-static struct osfp_os_class_score *osfp_score_db_array_match(void *data, void *value, unsigned int len)
+static struct osfp_field_value_count *osfp_score_db_array_match(void *data, void *value, unsigned int len)
 {
     unsigned int index;
     struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data;
@@ -84,7 +93,7 @@ static void *osfp_score_db_array_create(void)
         return NULL;
     }
 
-    array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_os_class_score));
+    array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_field_value_count));
     if (array_data->array_head == NULL) {
         free(array_data);
         return NULL;
@@ -106,13 +115,13 @@ static void osfp_score_db_array_destroy(void *data) {
     }
 }
 
-static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len)
+static int osfp_score_db_hash_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len)
 {
     int ret = -1, i;
     struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data;
     struct osfp_score_db_hash_element *element = NULL;
 
-    if (hash_data == NULL || os_class_score == NULL || value == NULL || len == 0) {
+    if (hash_data == NULL || fvc == NULL || value == NULL || len == 0) {
         goto exit;
     }
 
@@ -124,8 +133,8 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla
         }
         element->key = strdup(value);
         element->keylen = len;
-        element->score = (struct osfp_os_class_score *)calloc(1, sizeof(struct osfp_os_class_score));
+        element->fvc = (struct osfp_field_value_count *)calloc(1, sizeof(struct osfp_field_value_count));
-        if (element->score == NULL) {
+        if (element->fvc == NULL) {
             free(element);
             element = NULL;
             goto exit;
@@ -134,7 +143,7 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla
     }
 
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        element->score->scores[i] += os_class_score->scores[i];
+        element->fvc->counts[i] += fvc->counts[i];
     }
 
     return 0;
@@ -142,7 +151,7 @@ exit:
     return ret;
 }
 
-static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *value, unsigned int len)
+static struct osfp_field_value_count *osfp_score_db_hash_match(void *data, void *value, unsigned int len)
 {
     struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data;
     struct osfp_score_db_hash_element *element = NULL;
@@ -160,7 +169,7 @@ static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *va
         return NULL;
     }
 
-    return element->score;
+    return element->fvc;
 }
 
 static void *osfp_score_db_hash_create(void)
@@ -181,8 +190,8 @@ static void osfp_score_db_hash_destroy(void *data) {
                     if (element->key) {
                         free(element->key);
                     }
-                    if (element->score) {
+                    if (element->fvc) {
-                        free(element->score);
+                        free(element->fvc);
                     }
                     free(element);
                 }
@@ -192,10 +201,136 @@ static void osfp_score_db_hash_destroy(void *data) {
     }
 }
 
+void osfp_score_db_debug_print(struct osfp_score_db *score_db)
+{
+    int i;
+    printf("score_db:\n");
+    printf("entry_count: %u\n", score_db->entry_count);
+    printf("total_weight: %u\n", score_db->total_weight);
+
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        const char *name = osfp_os_class_id_to_name(i);
+        printf("os class %s ", name);
+
+        printf("entry_count: %u\n", score_db->os_entry_count[i]);
+        printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_entry_count[i]);
+    }
+
+    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+        printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled);
+        printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type);
+        printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count);
+    }
+}
+
+const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
+{
+    int ret, i;
+    unsigned int value_buffer_used = 0;
+    char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX];
+    struct osfp_prefilter_hash_element *element = NULL;
+
+    if (score_db->prefilter_head == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            continue;
+        }
+        if (fp->fields[i].value && fp->fields[i].value_len != 0) {
+            memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len);
+            value_buffer_used += fp->fields[i].value_len;
+        }
+    }
+
+    HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element);
+    if (element == NULL) {
+        return NULL;
+    }
+
+    if (element->repeated) {
+        return NULL;
+    }
+
+    memset(result_score, 0, sizeof(struct osfp_os_class_score));
+    result_score->scores[element->os_class] = OSFP_PERCENTILE;
+
+    return (const char *)element->fp_json;
+}
+
+int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
+{
+    int ret = -1, i, j;
+
+    unsigned int coefficient;
+    unsigned int matched_count;
+    unsigned int os_entry_count;
+    unsigned int total_weight;
+    unsigned int field_weight;
+
+    struct osfp_fingerprint_field *field;
+    struct osfp_field_value_count *fvc; // field_value_count
+    struct osfp_field_score_db *fdb;
+
+    if (score_db == NULL || fp == NULL || result_score == NULL) {
+        goto exit;
+    }
+
+    // score
+    memset(result_score, 0, sizeof(struct osfp_os_class_score));
+
+    total_weight = score_db->total_weight;
+    if (total_weight == 0) {
+        goto exit;
+    }
+
+    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+        fdb = &score_db->field_score_dbs[i];
+        if (!fdb->enabled) {
+            continue;
+        }
+
+        field = &fp->fields[i];
+        if (!field->enabled) {
+            continue;
+        }
+
+        fvc = fdb->match(fdb->data, field->value, field->value_len);
+        if (fvc == NULL) {
+            continue;
+        }
+
+        coefficient = OSFP_PERCENTILE / PERFECT_SCORE_EXPECTED_RATE * score_db->field_weight[i] / total_weight;
+
+        for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
+            os_entry_count = score_db->os_entry_count[j];
+            matched_count = MIN(fvc->counts[j], os_entry_count * FIELD_VALUE_DUP_RATE_MAX);
+
+            if (os_entry_count == 0 || matched_count == 0) {
+                continue;
+            }
+
+            if (0 == flags || flags & OSFP_BIT_U32(j)) {
+                result_score->scores[j] += coefficient * matched_count / os_entry_count;
+            }
+        }
+
+        // if tcp options matched tcp options ordered is not needed
+        if (i == OSFP_FIELD_TCP_OPTIONS) {
+            i++;
+        }
+    }
+
+    return 0;
+exit:
+    return ret;
+}
+
 static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field, enum osfp_os_class_id os_class)
 {
     int ret = -1;
-    struct osfp_os_class_score os_class_score = {0};
+    struct osfp_field_value_count field_value_count = {0};
 
     void *value_ptr;
     unsigned int value_len;
@@ -216,9 +351,9 @@ static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field
             goto exit;
     }
 
-    os_class_score.scores[os_class] = 1;
+    field_value_count.counts[os_class] = 1;
 
-    ret = db->add(db->data, &os_class_score, value_ptr, value_len);
+    ret = db->add(db->data, &field_value_count, value_ptr, value_len);
     if (ret != 0) {
         goto exit;
     }
@@ -241,6 +376,7 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
         goto exit;
     }
 
+    // get os class
     field = cJSON_GetObjectItem(entry, osfp_fingerprint_get_field_name(OSFP_FIELD_OS));
     if (field == NULL || field->valuestring == NULL) {
         goto exit;
@@ -301,247 +437,58 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
     }
 
     score_db->entry_count++;
-    score_db->os_class_entry_count[os_class]++;
+    score_db->os_entry_count[os_class]++;
 
     return 0;
 exit:
     return ret;
 }
 
-char *osfp_score_db_read_file(char *fp_file)
-{
-    int ret = -1;
-    char *file_buffer = NULL;
-    unsigned int file_len = 0;
-    FILE *fp = NULL;
-    struct stat st;
-
-    if (0 > stat(fp_file, &st)) {
-        osfp_log_error("stat() failed on '%s'.\n", fp_file);
-        goto exit;
-    }
-
-    if (st.st_size == 0) { 
-        osfp_log_error("Empty file: '%s'.\n", fp_file);
-        goto exit;
-    }
-
-    file_len = (unsigned int)st.st_size;
-    file_buffer = malloc(file_len + 1);
-    if (file_buffer == NULL) {
-        osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file);
-        goto exit;
-    }
-
-    fp = fopen(fp_file, "r");
-    if (fp == NULL) {
-        osfp_log_error("Cannot open '%s' for reading.\n", fp_file);
-        goto exit;
-    }
-
-    ret = fread(file_buffer, 1, file_len, fp);
-    if (ret < 0) {
-        osfp_log_error("fread() failed on '%s'.\n", fp_file);
-        free(file_buffer);
-        fclose(fp);
-        goto exit;
-    }
-
-    fclose(fp);
-
-    file_buffer[file_len] = 0;
-
-    return file_buffer;
-exit:
-    return NULL;
-}
-
 int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file)
 {
-    int ret = OSFP_EINVAL, i, count;
+    int ret = -1, i, count;
     char *file_buffer;
-    cJSON *root = NULL;
     cJSON *entry;
+    cJSON *root = NULL;
     struct osfp_field_score_db *field_score_db;
 
-    if (score_db == NULL) {
+    if (score_db == NULL || fp_file == NULL) {
         goto exit;
     }
 
-    file_buffer = osfp_score_db_read_file(fp_file);
+    file_buffer = osfp_read_file(fp_file);
     if (file_buffer == NULL) {
         osfp_log_error("read file: '%s'\n", fp_file);
-        ret = OSFP_ERR_SCORE_DB_READ_FILE;
         goto exit;
     }
 
     root = cJSON_Parse(file_buffer);
     if (root == NULL) {
         osfp_log_error("parse json: '%s'\n", fp_file);
-        ret = OSFP_ERR_SCORE_DB_PARSE_FILE;
         goto exit;
     }
 
     count = cJSON_GetArraySize(root);
-
     for (i = 0; i < count; i++) {
         entry = cJSON_GetArrayItem(root, i);
-        if (entry) {
+        ret = osfp_score_db_load_entry(score_db, entry);
-            ret = osfp_score_db_load_entry(score_db, entry);
+        if (ret != 0) {
-            if (ret != 0) {
+            osfp_log_debug("json entry load failed.\n%s\n", cJSON_Print(entry));
-                osfp_log_debug("json entry load failed.\n%s\n", cJSON_Print(entry));
+            continue;
-                continue;
-            }
         }
     }
 
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
+    ret = 0;
-        field_score_db = &score_db->field_score_dbs[i];
-        if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS_ORDERED) {
-            score_db->perfect_score += osfp_fingerprint_get_field_importance(i);
-        }
-    }
-
-    score_db->perfect_score = score_db->perfect_score * PERFECT_SCORE_EXPECTED_RATE;
-
-    ret = OSFP_NOERR;
 exit:
     if (root) {
         cJSON_Delete(root);
     }
     if (file_buffer) {
-        free((char*)file_buffer);
+        free(file_buffer);
     }
     return ret;
 }
 
-int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
-{
-    int ret, i;
-    unsigned int value_buffer_used = 0;
-    char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX];
-    struct osfp_prefilter_hash_element *element = NULL;
-
-    if (score_db->prefilter_head == NULL) {
-        return 0;
-    }
-
-    for (i = 0; i < OSFP_FIELD_OS; i++) {
-        if (0 == osfp_fingerprint_get_field_enabled(i)) {
-            continue;
-        }
-        if (fp->fields[i].value && fp->fields[i].value_len != 0) {
-            memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len);
-            value_buffer_used += fp->fields[i].value_len;
-        }
-    }
-
-    HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element);
-    if (element == NULL) {
-        return 0;
-    }
-
-    if (element->repeated) {
-        return 0;
-    }
-
-    memset(result_score, 0, sizeof(struct osfp_os_class_score));
-    result_score->scores[element->os_class] = OSFP_PERCENTILE;
-
-    return 1;
-}
-
-int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
-{
-    int ret = OSFP_EINVAL, i, j;
-
-    unsigned int tmp_score;
-    unsigned int perfect_score;
-    unsigned int entry_count;
-    unsigned int importance;
-
-    struct osfp_os_class_score *os_class_score_matched;
-    enum osfp_os_class_id os_class_id;
-
-    struct osfp_fingerprint_field *field;
-    struct osfp_field_score_db *field_score_db;
-
-    if (score_db == NULL || fp == NULL || result_score == NULL) {
-        goto exit;
-    }
-
-    // score
-    memset(result_score, 0, sizeof(struct osfp_os_class_score));
-
-    perfect_score = score_db->perfect_score;
-    if (perfect_score == 0) {
-        goto exit;
-    }
-
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
-        field_score_db = &score_db->field_score_dbs[i];
-        if (!field_score_db->enabled) {
-            continue;
-        }
-
-        field = &fp->fields[i];
-        if (!field->enabled) {
-            continue;
-        }
-
-        os_class_score_matched = field_score_db->match(field_score_db->data, field->value, field->value_len);
-        if (os_class_score_matched == NULL) {
-            continue;
-        }
-
-        importance = osfp_fingerprint_get_field_importance(i);
-
-        for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
-            entry_count = score_db->os_class_entry_count[j];
-            tmp_score = os_class_score_matched->scores[j];
-            tmp_score = tmp_score < (entry_count * FIELD_VALUE_DUP_RATE_MAX) ? tmp_score : (entry_count * FIELD_VALUE_DUP_RATE_MAX);
-            if (entry_count == 0 || tmp_score == 0) {
-                continue;
-            }
-            if (0 == flags || flags & OSFP_BIT_U32(j)) {
-                osfp_log_debug("%s %s: ((%d * %u / %u) * %u ) / %u\n", osfp_fingerprint_get_field_name(i), osfp_os_class_id_to_name(j), OSFP_PERCENTILE, importance, perfect_score, os_class_score_matched->scores[j], entry_count);
-                result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * tmp_score) / entry_count;
-            }
-        }
-
-        if (i == OSFP_FIELD_TCP_OPTIONS) {
-            // if OSFP_FIELD_TCP_OPTIONS matched OSFP_FIELD_TCP_OPTIONS_ORDERED is not needed
-            i++;
-        }
-    }
-
-    return OSFP_NOERR;
-exit:
-    return ret;
-}
-
-void osfp_score_db_debug_print(struct osfp_score_db *score_db)
-{
-    int i;
-    printf("score_db:\n");
-    printf("entry_count: %u\n", score_db->entry_count);
-    printf("perfect_score: %u\n", score_db->perfect_score);
-
-    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
-        const char *name = osfp_os_class_id_to_name(i);
-        printf("os class %s ", name);
-
-        printf("entry_count: %u\n", score_db->os_class_entry_count[i]);
-        printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_class_entry_count[i]);
-    }
-
-    for (i = 0; i < OSFP_FIELD_MAX; i++) {
-        printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled);
-        printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type);
-        printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count);
-    }
-}
-
 struct osfp_score_db *osfp_score_db_create(void)
 {
     int i;
@@ -562,6 +509,12 @@ struct osfp_score_db *osfp_score_db_create(void)
             continue;
         }
 
+        score_db->field_weight[i] = osfp_fingerprint_get_field_importance(i);
+        // tcp options ordered and tcp options overlap
+        if (i != OSFP_FIELD_TCP_OPTIONS_ORDERED) {
+            score_db->total_weight += score_db->field_weight[i];
+        }
+
         db->type = osfp_fingerprint_get_field_type(i);
         switch (db->type) {
             case OSFP_FIELD_TYPE_UINT:
@@ -577,13 +530,13 @@ struct osfp_score_db *osfp_score_db_create(void)
                 db->match = osfp_score_db_hash_match;
                 break;
             default:
-                osfp_log_debug("fingerprint field unsupported type: %u", db->type);
+                osfp_log_error("fingerprint field unsupported type: %u", db->type);
                 goto exit;
         }
 
         db->data = db->create();
         if (db->data == NULL) {
-            osfp_log_debug("field db create failed. field: %s", osfp_fingerprint_get_field_name(i));
+            osfp_log_error("field db create failed. field: %s", osfp_fingerprint_get_field_name(i));
             goto exit;
         }
     }
@@ -680,7 +633,7 @@ int test_osfp_score_db(void)
         goto exit;
     }
 
-    if (db->os_class_entry_count[OSFP_OS_CLASS_ANDROID] != 1) {
+    if (db->os_entry_count[OSFP_OS_CLASS_ANDROID] != 1) {
         goto exit;
     }
 

src/osfp_score_db.h

@@ -9,6 +9,10 @@ struct osfp_os_class_score {
     unsigned int scores[OSFP_OS_CLASS_MAX];
 };
 
+struct osfp_field_value_count {
+    unsigned int counts[OSFP_OS_CLASS_MAX];
+};
+
 struct osfp_field_score_db {
     unsigned int    enabled;
     unsigned int    type;
@@ -18,34 +22,38 @@ struct osfp_field_score_db {
 
     void *(*create)(void);
     void (*destroy)(void *);
-    int (*add)(void *data, struct osfp_os_class_score *, void *, unsigned int);
+    int (*add)(void *data, struct osfp_field_value_count *, void *, unsigned int);
-    struct osfp_os_class_score *(*match)(void *, void *, unsigned int);
+    struct osfp_field_value_count *(*match)(void *, void *, unsigned int);
 };
 
 struct osfp_prefilter_hash_element {
-    struct osfp_fingerprint *fp;
-    char *fp_json;
     unsigned int os_class;
-
     unsigned int repeated;
+
+    struct osfp_fingerprint *fp;
+    char                    *fp_json;
+
     UT_hash_handle hh;
 };
 
 struct osfp_score_db {
-    unsigned int entry_count;
-
     struct osfp_prefilter_hash_element *prefilter_head;
 
+    unsigned int entry_count;
+    unsigned int os_entry_count[OSFP_OS_CLASS_MAX];
+
     unsigned int perfect_score;
-    unsigned int os_class_entry_count[OSFP_OS_CLASS_MAX];
+    unsigned int total_weight;
+    unsigned int field_weight[OSFP_FIELD_MAX];
+
     struct osfp_field_score_db field_score_dbs[OSFP_FIELD_MAX];
 };
 
-char *osfp_score_db_read_file(char *fp_file);
 void osfp_score_db_debug_print(struct osfp_score_db *score_db);
 
+const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
+
 int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file);
-int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
 int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
 
 struct osfp_score_db *osfp_score_db_create(void);

test/test.c

@@ -132,7 +132,7 @@ void test_data_prepare()
     char *file_buffer;
 
     if (test_file_path == NULL) {
-        file_buffer = osfp_score_db_read_file(data_file_path);
+        file_buffer = osfp_read_file(data_file_path);
         if (file_buffer == NULL) {
             osfp_log_error("read file: '%s'\n", data_file_path);
             exit(1);
@@ -188,7 +188,7 @@ void test_miss_rate()
     unsigned int fingerprint_count = 0;
     char *file_buffer;
 
-    file_buffer = osfp_score_db_read_file(test_file_path);
+    file_buffer = osfp_read_file(test_file_path);
     if (file_buffer == NULL) {
         osfp_log_error("read file: '%s'\n", test_file_path);
         exit(1);