test: show prefilter stats

prefilter: fix fields buffered order
score_db: support hash prefilter
2023-10-23 14:57:38 +08:00 · 2023-10-23 14:52:21 +08:00 · 2023-10-23 09:54:02 +08:00 · 2023-10-23 09:52:46 +08:00 · 2023-10-23 09:52:21 +08:00 · 2023-10-17 23:37:20 +08:00
18 changed files with 656 additions and 298 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -98,7 +98,7 @@ develop_build_debug_for_centos7:
    PULP3_REPO_NAME: framework-testing-x86_64.el7
    PULP3_DIST_NAME: framework-testing-x86_64.el7
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-debug"
+    name: "libosfp-$CI_COMMIT_REF_NAME-debug"
    paths:
      - build/*.rpm
  only:
@@ -118,7 +118,7 @@ develop_build_release_for_centos7:
    PULP3_REPO_NAME: framework-testing-x86_64.el7
    PULP3_DIST_NAME: framework-testing-x86_64.el7
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-release"
+    name: "libosfp-$CI_COMMIT_REF_NAME-release"
    paths:
      - build/*.rpm
  only:
@@ -137,7 +137,7 @@ release_build_debug_for_centos7:
    PULP3_DIST_NAME: framework-stable-x86_64.el7
  extends: .build_by_travis_for_centos7
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-debug"
+    name: "libosfp-$CI_COMMIT_REF_NAME-debug"
    paths:
      - build/*.rpm
  only:
@@ -153,7 +153,7 @@ release_build_release_for_centos7:
    PULP3_DIST_NAME: framework-stable-x86_64.el7
  extends: .build_by_travis_for_centos7
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-release"
+    name: "libosfp-$CI_COMMIT_REF_NAME-release"
    paths:
      - build/*.rpm
  only:
@@ -195,7 +195,7 @@ develop_build_debug_for_centos8:
    PULP3_REPO_NAME: framework-testing-x86_64.el8
    PULP3_DIST_NAME: framework-testing-x86_64.el8
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-debug"
+    name: "libosfp-$CI_COMMIT_REF_NAME-debug"
    paths:
      - build/*.rpm
  only:
@@ -215,7 +215,7 @@ develop_build_release_for_centos8:
    PULP3_REPO_NAME: framework-testing-x86_64.el8
    PULP3_DIST_NAME: framework-testing-x86_64.el8
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-release"
+    name: "libosfp-$CI_COMMIT_REF_NAME-release"
    paths:
      - build/*.rpm
  only:
@@ -234,7 +234,7 @@ release_build_debug_for_centos8:
    PULP3_DIST_NAME: framework-stable-x86_64.el8
  extends: .build_by_travis_for_centos8
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-debug"
+    name: "libosfp-$CI_COMMIT_REF_NAME-debug"
    paths:
      - build/*.rpm
  only:
@@ -250,7 +250,7 @@ release_build_release_for_centos8:
    PULP3_DIST_NAME: framework-stable-x86_64.el8
  extends: .build_by_travis_for_centos8
  artifacts:
-    name: "MESA_osfp-$CI_COMMIT_REF_NAME-release"
+    name: "libosfp-$CI_COMMIT_REF_NAME-release"
    paths:
      - build/*.rpm
  only:
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,7 +5,7 @@ set(lib_name osfp)
 project (${lib_name})

 set(LIB_MAJOR_VERSION 1)
-set(LIB_MINOR_VERSION 2)
+set(LIB_MINOR_VERSION 3)

 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 include(Version)
@@ -55,11 +55,11 @@ set_target_properties(${lib_name}_static PROPERTIES OUTPUT_NAME ${lib_name})

 set(CMAKE_INSTALL_PREFIX /opt/MESA)

-install(FILES src/MESA_osfp.h   DESTINATION
+install(FILES src/osfp.h   DESTINATION
    ${CMAKE_INSTALL_PREFIX}/include/MESA COMPONENT devel)
 install(TARGETS ${lib_name}_shared  LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)
-install(FILES src/MESA_osfp.h   DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA COMPONENT HEADER)
-install(FILES fp.json DESTINATION /var/lib/MESA_osfp COMPONENT PROFILE)
+install(FILES src/osfp.h   DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA COMPONENT HEADER)
+install(FILES fp.json DESTINATION /var/lib/libosfp COMPONENT PROFILE)

 add_executable(${lib_name}_sample example/sample.c)
 target_link_libraries(${lib_name}_sample ${lib_name}_shared)
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,2 +0,0 @@
-SUBDIRS = src example test
-ACLOCAL_AMFLAGS=-I m4
--- a/README.md
+++ b/README.md
@@ -21,5 +21,5 @@ LD_LIBRARY_PATH=${PWD}/build ./sample

 ```
 # load the fingerprint file and capture on eth0, filter tcp port 8888
-./build/osfp_example -f /var/lib/MESA_osfp/fp.json -i eth0 "tcp port 8888"
+./build/osfp_example -f /var/lib/libosfp/fp.json -i eth0 "tcp port 8888"
 ```
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,14 +0,0 @@
-#!/bin/sh
-# Run this to generate all the initial makefiles, etc.
-if which libtoolize > /dev/null; then
-  echo "Found libtoolize"
-  libtoolize -c
-elif which glibtoolize > /dev/null; then
-  echo "Found glibtoolize"
-  glibtoolize -c
-else
-  echo "Failed to find libtoolize or glibtoolize, please ensure it is installed and accessible via your PATH env variable"
-  exit 1
-fi;
-autoreconf -ifv || exit 1
-echo "You can now run \"./configure\" and then \"make\"."
--- a/example/osfp_example.c
+++ b/example/osfp_example.c
@@ -10,7 +10,7 @@
 #include <pcap.h>

 #include "osfp_common.h"
-#include "MESA_osfp.h"
+#include "osfp.h"
 #include "osfp_log.h"
 #include "osfp_fingerprint.h"
 #include "osfp_score_db.h"
@@ -431,7 +431,7 @@ const char *PrintInet(int af, const void *src, char *dst, socklen_t size)
 void example_detect(struct osfp_db *osfp_db, Packet *p)
 {
    int ret;
-    char str_buf[1024];
+    char str_buf[2048] = "";
    //unsigned char *iph = (unsigned char *)(p->iph != NULL ? (void *)p->iph : (void *)p->ip6h);
    struct iphdr *iph;
    struct ip6_hdr *ip6h;
@@ -448,11 +448,20 @@ void example_detect(struct osfp_db *osfp_db, Packet *p)
    osfp_profile_cycle(c2);


+    struct osfp_fingerprint fp = {0};
+    if (iph) {
+        osfp_fingerprinting((unsigned char*)iph, (unsigned char*)tcph, tcph_len, &fp, 4);
+    } else if (ip6h) {
+        osfp_fingerprinting((unsigned char*)iph, (unsigned char*)tcph, tcph_len, &fp, 6);
+    } else {
+        goto exit;
+    }
+
    osfp_profile_get_cycle(c1);
    if (iph) {
-        result = MESA_osfp_ipv4_identify(osfp_db, iph, tcph, tcph_len);
+        result = osfp_ipv4_identify(osfp_db, iph, tcph, tcph_len);
    } else if (ip6h) {
-        result = MESA_osfp_ipv6_identify(osfp_db, ip6h, tcph, tcph_len);
+        result = osfp_ipv6_identify(osfp_db, ip6h, tcph, tcph_len);
    } else {
        goto exit;
    }
@@ -469,19 +478,27 @@ void example_detect(struct osfp_db *osfp_db, Packet *p)

    result_os_count[result->likely_os_class]++;

-    char *json = MESA_osfp_result_score_detail_export(result);
+    char *json = osfp_result_score_detail_export(result);

-    if (1) {
+    if (debug_enable) {
+        if (p->tcph->ack) {
+            printf("---------------------------  SYN/ACK\n");
+        } else {
+            printf("---------------------------  SYN\n");
+        }
+        osfp_fingerprint_to_json_buf(&fp, str_buf, 2048, 0);
+        printf("%s\n", str_buf);
        printf("Example ipv4 header detect: --------------------------\n");
        printf("Connection info: %s:%d -> %s:%d\n", p->srcip, p->sp, p->dstip, p->dp);
-        printf("Most likely os class: %s\n", MESA_osfp_result_os_name_get(result));
+        printf("Most likely os class: %s\n", osfp_result_os_name_get(result));
        printf("Details:\n");
        printf("%s\n", json);
+        fflush(stdout);
    }

 exit:
    if (result) {
-        MESA_osfp_result_free(result);
+        osfp_result_free(result);
    }
    return;
 }
@@ -503,11 +520,6 @@ void process_packet(char *user, struct pcap_pkthdr *h, u_char *pkt)
        goto exit;
    }

-    if (p->tcph->ack) {
-        printf("---------------------------  SYN/ACK\n");
-    } else {
-        printf("---------------------------  SYN\n");
-    }

    if (p->iph) {
        PrintInet(AF_INET, (const void *)&(p->src.addr_data32[0]), p->srcip, sizeof(p->srcip));
@@ -548,6 +560,8 @@ static void signal_handler(int signum)
        printf("%s: %u\n", osfp_os_class_id_to_name(i), result_os_count[i]);
    }

+    fflush(stdout);
+
    exit(0);
 }

@@ -670,7 +684,7 @@ int main(int argc, char *argv[])

    osfp_profile_set(1);

-    struct osfp_db *osfp_db = MESA_osfp_db_new(fp_file_path);
+    struct osfp_db *osfp_db = osfp_db_new(fp_file_path);
    if (osfp_db == NULL) {
        printf("could not create osfp context. fingerprints file: %s\n", fp_file_path);
        exit(1);
@@ -686,7 +700,7 @@ int main(int argc, char *argv[])
    }

    // destroy osfp db
-    MESA_osfp_db_free(osfp_db);
+    osfp_db_free(osfp_db);

    return 0;
 }
--- a/example/sample.c
+++ b/example/sample.c
@@ -1,5 +1,5 @@
 #include "stdio.h" 
-#include "MESA_osfp.h" 
+#include "osfp.h" 
 
 char iph[] = { 
  0x45, 0x00, 0x00, 0x34, 0x51, 0xc4, 0x40, 0x00, 
@@ -22,14 +22,14 @@ int main(int argc, char **argv)
    struct tcphdr *l4_hdr = (struct tcphdr *)tcph; 
    size_t l4_hdr_len = sizeof(tcph); 
 
-    struct osfp_db *db = MESA_osfp_db_new(json_file_path); 
+    struct osfp_db *db = osfp_db_new(json_file_path); 
    if (db) {
-        struct osfp_result *result = MESA_osfp_ipv4_identify(db, l3_hdr, l4_hdr, l4_hdr_len); 
+        struct osfp_result *result = osfp_ipv4_identify(db, l3_hdr, l4_hdr, l4_hdr_len); 
        if (result) { 
-            printf("likely os: %s\n", MESA_osfp_result_os_name_get(result)); 
-            printf("details: \n%s\n", MESA_osfp_result_score_detail_export(result)); 
-            MESA_osfp_result_free(result);
+            printf("likely os: %s\n", osfp_result_os_name_get(result)); 
+            printf("details: \n%s\n", osfp_result_score_detail_export(result)); 
+            osfp_result_free(result);
        } 
-        MESA_osfp_db_free(db); 
+        osfp_db_free(db); 
    }
 }
--- a/libosfp-config.in
+++ b/libosfp-config.in
@@ -1,4 +0,0 @@
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
--- a/src/MESA_osfp.c
+++ b/src/MESA_osfp.c
@@ -1,6 +1,6 @@
 #include "osfp_common.h"

-#include "MESA_osfp.h"
+#include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_score_db.h"
 #include "osfp_log.h"
@@ -43,7 +43,9 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
        }
    }

-    if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
+    if (likely_score == OSFP_PERCENTILE) {
+        ;
+    } else if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
        likely_os_class = OSFP_OS_CLASS_OTHERS;
    } else {
        for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
@@ -70,7 +72,7 @@ exit:
    return NULL;
 }

-const char *MESA_osfp_result_os_name_get(struct osfp_result *result)
+const char *osfp_result_os_name_get(struct osfp_result *result)
 {
    enum osfp_os_class_id os_class;

@@ -86,7 +88,7 @@ const char *MESA_osfp_result_os_name_get(struct osfp_result *result)
    return osfp_os_class_id_to_name(os_class);
 }

-char *MESA_osfp_result_score_detail_export(struct osfp_result *result)
+char *osfp_result_score_detail_export(struct osfp_result *result)
 {
    int i;
    char *result_str = NULL;
@@ -150,7 +152,7 @@ exit:
    return result_str;
 }

-void MESA_osfp_result_free(struct osfp_result *result)
+void osfp_result_free(struct osfp_result *result)
 {
    if (result) {
        if (result->json_str) {
@@ -160,7 +162,7 @@ void MESA_osfp_result_free(struct osfp_result *result)
    }
 }

-struct osfp_result *MESA_osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
+struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
 {
    int ret = OSFP_EINVAL;
    struct osfp_fingerprint fp;
@@ -183,11 +185,17 @@ struct osfp_result *MESA_osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3
    }

    osfp_profile_get_cycle(c1);
-    ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+    ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
    osfp_profile_get_cycle(c2);
-    osfp_profile_counter_update(&osfp_profile_score, c2 - c1);
-    if (ret != 0) {
-        goto exit;
+    osfp_profile_counter_update(&osfp_profile_prefilter, c2 - c1);
+    if (ret <= 0) {
+        osfp_profile_get_cycle(c1);
+        ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+        osfp_profile_get_cycle(c2);
+        osfp_profile_counter_update(&osfp_profile_score, c2 - c1);
+        if (ret != 0) {
+            goto exit;
+        }
    }

    osfp_profile_get_cycle(c1);
@@ -203,7 +211,7 @@ exit:
    return NULL;
 }

-struct osfp_result *MESA_osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
+struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len)
 {
    int ret = OSFP_EINVAL;
    struct osfp_fingerprint fp;
@@ -234,7 +242,7 @@ exit:
    return NULL;
 }

-struct osfp_result *MESA_osfp_json_identify(struct osfp_db *db, const char *json_str)
+struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str)
 {
    int ret = OSFP_EINVAL;
    struct osfp_fingerprint fp;
@@ -250,9 +258,12 @@ struct osfp_result *MESA_osfp_json_identify(struct osfp_db *db, const char *json
        goto exit;
    }

-    ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
-    if (ret != 0) {
-        goto exit;
+    ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score);
+    if (ret <= 0) {
+        ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score);
+        if (ret != 0) {
+            goto exit;
+        }
    }

    result = osfp_result_build(&os_class_score);
@@ -265,7 +276,7 @@ exit:
    return NULL;
 }

-struct osfp_db *MESA_osfp_db_new(const char *db_json_file)
+struct osfp_db *osfp_db_new(const char *db_json_file)
 {
    int ret;
    struct osfp_db *db;
@@ -299,12 +310,12 @@ struct osfp_db *MESA_osfp_db_new(const char *db_json_file)
    return db;
 exit:
    if (db) {
-        MESA_osfp_db_free(db);
+        osfp_db_free(db);
    }
    return NULL;
 }

-void MESA_osfp_db_free(struct osfp_db *db)
+void osfp_db_free(struct osfp_db *db)
 {
    if (db) {
        if (db->db_json_path) {
--- a/src/MESA_osfp.h
+++ b/src/MESA_osfp.h
@@ -23,14 +23,14 @@ struct osfp_db;
 * @param db_json_path 操作系统指纹库 JSON 文件的路径。
 * @return 指向新创建的操作系统指纹库的指针。
 */
-struct osfp_db *MESA_osfp_db_new(const char *db_json_path);
+struct osfp_db *osfp_db_new(const char *db_json_path);

 /**
 * @brief 释放操作系统指纹库占用的内存。
 * 
 * @param db 指向要释放的操作系统指纹库的指针。
 */
-void MESA_osfp_db_free(struct osfp_db *db);
+void osfp_db_free(struct osfp_db *db);

 /**
 * @brief 通过 IPv4 头部和 TCP 头部识别操作系统。
@@ -41,7 +41,7 @@ void MESA_osfp_db_free(struct osfp_db *db);
 * @param l4_hdr_len TCP 头部的长度（注意：包含TCP选项部分）。
 * @return 指向操作系统识别结果的指针。
 */
-struct osfp_result *MESA_osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len);
+struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len);

 /**
 * @brief 通过 IPv6 头部和 TCP 头部识别操作系统。
@@ -52,7 +52,7 @@ struct osfp_result *MESA_osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3
 * @param l4_hdr_len TCP 头部的长度（注意：包含TCP选项部分）。
 * @return 指向操作系统识别结果的指针（注意：内存需要使用者释放）。
 */
-struct osfp_result *MESA_osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len);
+struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len);

 /**
 * @brief 通过 json 格式的指纹识别操作系统。
@@ -61,7 +61,7 @@ struct osfp_result *MESA_osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr*
 * @param json_str 指纹字符串。
 * @return 指向操作系统识别结果的指针（注意：内存需要使用者释放）。
 */
-struct osfp_result *MESA_osfp_json_identify(struct osfp_db *db, const char *json_str);
+struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str);

 /**
 * @brief 获取操作系统识别结果的操作系统名称。
@@ -69,7 +69,7 @@ struct osfp_result *MESA_osfp_json_identify(struct osfp_db *db, const char *json
 * @param result 操作系统识别结果。
 * @return 指向操作系统名称的常量字符指针（注意：这块内存将由osfp_result_free释放）。
 */
-const char *MESA_osfp_result_os_name_get(struct osfp_result *result);
+const char *osfp_result_os_name_get(struct osfp_result *result);

 /**
 * @brief 导出操作系统识别结果的得分详情。
@@ -77,14 +77,14 @@ const char *MESA_osfp_result_os_name_get(struct osfp_result *result);
 * @param result 操作系统识别结果。
 * @return 指向得分详情字符串的指针（注意：内存需要使用者释放）。
 */
-char *MESA_osfp_result_score_detail_export(struct osfp_result *result);
+char *osfp_result_score_detail_export(struct osfp_result *result);

 /**
 * @brief 释放操作系统识别结果占用的内存。
 * 
 * @param result 操作系统识别结果。
 */
-void MESA_osfp_result_free(struct osfp_result *result);
+void osfp_result_free(struct osfp_result *result);

 #ifdef __cplusplus
 }
--- a/src/osfp_common.c
+++ b/src/osfp_common.c
@@ -1,6 +1,6 @@
 #include "osfp_common.h"

-#include "MESA_osfp.h"
+#include "osfp.h"

 unsigned int osfp_profile_enable;

@@ -15,6 +15,7 @@ const char *osfp_os_class_name[OSFP_OS_CLASS_MAX] = {
 };

 struct osfp_profile_counter osfp_profile_fingerprinting;
+struct osfp_profile_counter osfp_profile_prefilter;
 struct osfp_profile_counter osfp_profile_score;
 struct osfp_profile_counter osfp_profile_result_build;
 struct osfp_profile_counter osfp_profile_result_export;
@@ -57,8 +58,9 @@ void osfp_profile_counter_update(struct osfp_profile_counter *profile, unsigned
 void osfp_profile_print_stats(void)
 {
    osfp_profile_counter_print(&osfp_profile_fingerprinting, "fingerprinting");
+    osfp_profile_counter_print(&osfp_profile_prefilter, "prefilter");
    osfp_profile_counter_print(&osfp_profile_score, "score");
-    osfp_profile_counter_print(&osfp_profile_result_build, "result_build");
+    osfp_profile_counter_print(&osfp_profile_result_build, "result build");
    osfp_profile_counter_print(&osfp_profile_result_export, "result export");
 }

--- a/src/osfp_common.h
+++ b/src/osfp_common.h
@@ -21,6 +21,8 @@

 #include "cJSON.h"

+#include "osfp.h"
+
 static inline unsigned long long osfp_rdtsc(void)
 {
 	union {
@@ -57,6 +59,7 @@ struct osfp_profile_counter {
 };

 extern struct osfp_profile_counter osfp_profile_fingerprinting;
+extern struct osfp_profile_counter osfp_profile_prefilter;
 extern struct osfp_profile_counter osfp_profile_score;
 extern struct osfp_profile_counter osfp_profile_result_build;
 extern struct osfp_profile_counter osfp_profile_result_export;
--- a/src/osfp_fingerprint.c
+++ b/src/osfp_fingerprint.c
@@ -1,6 +1,6 @@
 #include "osfp_common.h"

-#include "MESA_osfp.h"
+#include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_log.h"

@@ -21,6 +21,27 @@

 #define OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME           "OSFP_UNKNOWN"

+#define OSFP_FP_SET_FIELD(fp, field_id, v, l) do {                                    \
+        (fp)->fields[(field_id)].name = osfp_fingerprint_get_field_name(field_id);  \
+        (fp)->fields[(field_id)].enabled = 1;                                       \
+        if ((fp)->value_buffer_used + (l) <= sizeof((fp)->value_buffer)) {        \
+            memcpy(fp->value_buffer + (fp)->value_buffer_used, (v), (l));     \
+            (fp)->fields[(field_id)].value = (fp)->value_buffer + (fp)->value_buffer_used;  \
+            (fp)->fields[(field_id)].value_len = (l);                             \
+            (fp)->value_buffer_used += (l);                                       \
+        } else {                                                                    \
+            (fp)->fields[(field_id)].value = NULL;                                  \
+            (fp)->fields[(field_id)].value_len = 0;                                 \
+        }                                                                           \
+    } while (0)
+
+#define OSFP_FP_INIT_FIELD(fp, field_id) do {                                       \
+        (fp)->fields[(field_id)].name = osfp_fingerprint_get_field_name((field_id));  \
+        (fp)->fields[(field_id)].enabled = 0;                                       \
+        (fp)->fields[(field_id)].value = NULL;                                      \
+        (fp)->fields[(field_id)].value_len = 0;                                     \
+    } while (0)
+
 struct osfp_tcp_opt {
    unsigned char type;
    unsigned char len;
@@ -40,7 +61,7 @@ struct osfp_fingerprint_field fp_fields[OSFP_FIELD_MAX] = {
    {OSFP_FINGERPRINT_FIELD_NAME_TCP_FLAGS,               1, OSFP_FIELD_TYPE_UINT,    25,     NULL,    0},
    {OSFP_FINGERPRINT_FIELD_NAME_TCP_MSS,                 1, OSFP_FIELD_TYPE_UINT,    150,    NULL,    0},
    {OSFP_FINGERPRINT_FIELD_NAME_TCP_OPTIONS,             1, OSFP_FIELD_TYPE_STRING,  400,    NULL,    0},
-    {OSFP_FINGERPRINT_FIELD_NAME_TCP_OPTIONS_ORDERED,     0, OSFP_FIELD_TYPE_STRING,  250,    NULL,    0},
+    {OSFP_FINGERPRINT_FIELD_NAME_TCP_OPTIONS_ORDERED,     1, OSFP_FIELD_TYPE_STRING,  250,    NULL,    0},
    {OSFP_FINGERPRINT_FIELD_NAME_OS,                      0, OSFP_FIELD_TYPE_STRING,  0,      NULL,    0},
 };

@@ -88,30 +109,6 @@ static unsigned int compute_ip_ttl(unsigned int ip_ttl)
    return ip_ttl;
 }

-static void osfp_fingerprint_setup_field(struct osfp_fingerprint *fp, enum osfp_field_id field_id, void *value, unsigned int len)
-{
-    fp->fields[field_id].name = osfp_fingerprint_get_field_name(field_id);
-    fp->fields[field_id].enabled = 1;
-
-    if (fp->value_buffer_used + len <= sizeof(fp->value_buffer)) {
-        memcpy(fp->value_buffer + fp->value_buffer_used, value, len);
-        fp->fields[field_id].value = fp->value_buffer + fp->value_buffer_used;
-        fp->fields[field_id].value_len = len;
-        fp->value_buffer_used += len;
-    } else {
-        fp->fields[field_id].value = NULL;
-        fp->fields[field_id].value_len = 0;
-    }
-}
-
-static void osfp_fingerprint_init_field(struct osfp_fingerprint *fp, enum osfp_field_id field_id)
-{
-    fp->fields[field_id].name = osfp_fingerprint_get_field_name(field_id);
-    fp->fields[field_id].enabled = 0;
-    fp->fields[field_id].value = NULL;
-    fp->fields[field_id].value_len = 0;
-}
-
 static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned int max_opt_cnt, unsigned char *data, unsigned int len)
 {
    unsigned int offset = 0;
@@ -149,184 +146,37 @@ static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned i
    return tcp_opt_cnt;
 }

-static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int opt_len, struct osfp_fingerprint *fp)
+static char *osfp_fingerprint_tcp_options_to_ordered(char *tcp_options, unsigned int len)
 {
-    int ret,i;
+    int i;
+    char *tcp_options_ordered;
+    unsigned tcp_options_ordered_offset;
+    unsigned tcp_options_offset;

-    unsigned int tcp_mss;
-    unsigned int tcp_ws;
-    unsigned int tcp_ter;
-    unsigned int tcp_opt_cnt;
-    struct osfp_tcp_opt tcp_opts[OSFP_TCP_OPTMAX];
-
-    char options[OSFP_TCP_OPTLENMAX];
-    char options_ordered[OSFP_TCP_OPTLENMAX];
-    unsigned int offset = 0;
-    unsigned int maxoffset = sizeof(options) - 3; //for shortest "E,"
-    unsigned int ordered_offset = 0;
-    unsigned int ordered_maxoffset = sizeof(options_ordered) - 1;
-
-    if (opt_data == NULL || opt_len == 0 || fp == NULL) {
+    if (tcp_options == NULL && len == 0) {
        goto exit;
    }

-    tcp_opt_cnt = decode_tcp_options(tcp_opts, OSFP_TCP_OPTMAX, opt_data, opt_len);
+    tcp_options_ordered = malloc(len + 1);
+    if (tcp_options_ordered == NULL) {
+        goto exit;
+    }

-    for (i = 0; i < tcp_opt_cnt && offset < maxoffset && ordered_offset < ordered_maxoffset; i++) {
-        struct osfp_tcp_opt *opt = &tcp_opts[i];
-
-        char letter = option_to_ascii(opt->type);
-        options[offset++] = letter;
-        options_ordered[ordered_offset++] = letter;
-
-        switch (opt->type) {
-            case OSFP_TCP_OPT_EOL:
-            case OSFP_TCP_OPT_NOP:
-                break;
-            case OSFP_TCP_OPT_MSS:
-                if (opt->len != OSFP_TCP_OPT_MSS_LEN) {
-                    break;
-                }
-                tcp_mss = (unsigned int)ntohs(*(unsigned short *)opt->data);
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_MSS, &tcp_mss, sizeof(tcp_mss));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_mss);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_WSCALE:
-                if (opt->len != OSFP_TCP_OPT_WS_LEN) {
-                    break;
-                }
-                tcp_ws = (unsigned int)*(unsigned char *)opt->data;
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_WINDOW_SCALING, &tcp_ws, sizeof(tcp_ws));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ws);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_TIMESTAMP:
-                if (opt->len != OSFP_TCP_OPT_TS_LEN) {
-                    break;
-                }
-                tcp_ter = ntohl(*(unsigned int *)(opt->data + 4));
-                osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY, &tcp_ter, sizeof(tcp_ter));
-                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ter);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
-                break;
-            case OSFP_TCP_OPT_SACKOK:
-            case OSFP_TCP_OPT_SACK:
-            case OSFP_TCP_OPT_ECHO:
-            case OSFP_TCP_OPT_ECHOREPLY:
-            case OSFP_TCP_OPT_POCONN:
-            case OSFP_TCP_OPT_POSVC:
-                break;
-            default:
-                ret = snprintf(options + offset, sizeof(options), "%u", opt->type);
-                if (ret < 0 || offset + ret > maxoffset) {
-                    break;
-                }
-                offset += ret;
+    tcp_options_offset = 0;
+    tcp_options_ordered_offset = 0;
+    while(tcp_options_offset < len) {
+        if (isalpha(tcp_options[tcp_options_offset])) {
+            tcp_options_ordered[tcp_options_ordered_offset] = tcp_options[tcp_options_offset];
+            tcp_options_ordered_offset++;
        }
-
-        options[offset++] = ',';
-        options[offset] = 0;
-        options_ordered[ordered_offset] = 0;
+        tcp_options_offset++;
    }

-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OPTIONS, options, strlen(options) + 1);
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED, options_ordered, strlen(options_ordered) + 1);
+    tcp_options_ordered[tcp_options_ordered_offset] = 0;

-    return 0;
+    return tcp_options_ordered;
 exit:
-    return -1;
-}
-
-static int osfp_fingerprinting_tcp(struct tcphdr *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp)
-{
-    unsigned int tcp_off;
-    unsigned int tcp_window_size;
-    unsigned int tcp_flags;
-
-    if (tcph == NULL || tcph_len > OSFP_TCP_DATA_OFF_MAX || fp == NULL) {
-        goto exit;
-    }
-
-    tcp_off = tcph->doff << 2;
-    tcp_window_size = ntohs(tcph->window);
-    tcp_flags = *((unsigned char *)&tcph->ack_seq + 5);
-
-    if (tcp_off != tcph_len) {
-        goto exit;
-    }
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_OFF, &tcp_off, sizeof(tcp_off));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_WINDOW_SIZE, &tcp_window_size, sizeof(tcp_window_size));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_TCP_FLAGS, &tcp_flags, sizeof(tcp_flags));
-
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_TIMESTAMP);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_WINDOW_SCALING);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_MSS);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_OPTIONS);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED);
-
-    // tcp options
-    if (tcp_off > OSFP_TCP_HEADER_LEN) {
-        osfp_fingerprinting_tcp_option((unsigned char *)tcph + OSFP_TCP_HEADER_LEN, tcp_off - OSFP_TCP_HEADER_LEN, fp);
-    }
-
-    return 0;
-exit:
-    return -1;
-}
-
-static int osfp_fingerprinting_ipv4(struct iphdr *iph, struct osfp_fingerprint *fp)
-{
-    if (iph == NULL || fp == NULL) {
-        goto exit;
-    }
-
-    unsigned int ip_id = !!iph->id;
-    unsigned int ip_tos = iph->tos;
-    unsigned int ip_total_length = ntohs(iph->tot_len);
-    unsigned int ip_ttl = compute_ip_ttl(iph->ttl);
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_ID, &ip_id, sizeof(ip_id));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOS, &ip_tos, sizeof(ip_tos));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
-
-    return 0;
-exit:
-    return -1;
-}
-
-static int osfp_fingerprinting_ipv6(struct ip6_hdr *iph, struct osfp_fingerprint *fp)
-{
-    if (iph == NULL || fp == NULL) {
-        goto exit;
-    }
-
-    //unsigned int ip_id = 0;
-    //unsigned int ip_tos = 0;
-    unsigned int ip_total_length = OSFP_IPV6_HEADER_LEN + ntohs(iph->ip6_ctlun.ip6_un1.ip6_un1_plen);
-    unsigned int ip_ttl = compute_ip_ttl(iph->ip6_ctlun.ip6_un1.ip6_un1_hlim);
-
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
-
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_IP_ID);
-    osfp_fingerprint_init_field(fp, OSFP_FIELD_IP_TOS);
-
-    return 0;
-exit:
-    return -1;
+    return NULL;
 }

 int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format)
@@ -371,6 +221,72 @@ int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsi
    return strlen(strbuf) + 1;
 }

+struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root)
+{
+    int i;
+    cJSON *field;
+    void *value_ptr;
+    unsigned int value_len;
+    struct osfp_fingerprint *fp;
+
+    if (root == NULL) {
+        goto exit;
+    }
+
+    fp = calloc(1, sizeof(struct osfp_fingerprint));
+    if (fp == NULL) {
+        goto exit;
+    }
+
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
+        field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(i));
+        if (field == NULL) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
+        switch (field->type) {
+            case cJSON_Number:
+                value_ptr = (void *)&field->valueint;
+                value_len = sizeof(field->valueint);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
+                break;
+            case cJSON_String:
+                value_ptr = (void *)field->valuestring;
+                value_len = strlen(field->valuestring) + 1;
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                break;
+            case cJSON_NULL:
+                //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
+                break;
+            default:
+                goto exit;
+        }
+    }
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_OS);
+
+    return fp;
+exit:
+    if (fp) {
+        free(fp);
+    }
+    return NULL;
+}
+
 int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
 {
    int ret, i;
@@ -392,11 +308,25 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
        goto exit;
    }

+    field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(OSFP_FIELD_TCP_OPTIONS_ORDERED));
+    if (field == NULL) {
+        field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(OSFP_FIELD_TCP_OPTIONS));
+        if (field) {
+            char *tcp_options_ordered_str = osfp_fingerprint_tcp_options_to_ordered(field->valuestring, strlen(field->valuestring));
+            if (tcp_options_ordered_str) {
+                cJSON_AddItemToObject(root, osfp_fingerprint_get_field_name(OSFP_FIELD_TCP_OPTIONS_ORDERED),
+                        cJSON_CreateString(tcp_options_ordered_str));
+                free(tcp_options_ordered_str);
+            }
+        }
+    }
+
    for (i = 0; i < OSFP_FIELD_OS; i++) {
-        if (!fp_fields[i].enabled) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
            continue;
        }
-    
+
        field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(i));
        if (field == NULL) {
            goto exit;
@@ -406,12 +336,20 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
            case cJSON_Number:
                value_ptr = (void *)&field->valueint;
                value_len = sizeof(field->valueint);
-                osfp_fingerprint_setup_field(fp, i, value_ptr, value_len);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
                break;
            case cJSON_String:
                value_ptr = (void *)field->valuestring;
                value_len = strlen(field->valuestring) + 1;
-                osfp_fingerprint_setup_field(fp, i, value_ptr, value_len);
+                OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
                break;
            case cJSON_NULL:
                //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root));
@@ -421,9 +359,195 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
        }
    }

+    ret = 0;
+exit:
+    if (root) {
+        cJSON_Delete(root);
+    }
+    return ret;
+}
+
+static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int opt_len, struct osfp_fingerprint *fp)
+{
+    int ret,i;
+
+    unsigned int tcp_mss;
+    unsigned int tcp_ws;
+    unsigned int tcp_ts;
+    unsigned int tcp_ter;
+    unsigned int tcp_opt_cnt;
+    struct osfp_tcp_opt tcp_opts[OSFP_TCP_OPTMAX];
+
+    char options[OSFP_TCP_OPTLENMAX];
+    char options_ordered[OSFP_TCP_OPTLENMAX];
+    unsigned int offset = 0;
+    unsigned int maxoffset = sizeof(options) - 3; //for shortest "E,"
+    unsigned int ordered_offset = 0;
+    unsigned int ordered_maxoffset = sizeof(options_ordered) - 1;
+
+    if (opt_data == NULL || opt_len == 0 || fp == NULL) {
+        goto exit;
+    }
+
+    tcp_opt_cnt = decode_tcp_options(tcp_opts, OSFP_TCP_OPTMAX, opt_data, opt_len);
+
+    for (i = 0; i < tcp_opt_cnt && offset < maxoffset && ordered_offset < ordered_maxoffset; i++) {
+        struct osfp_tcp_opt *opt = &tcp_opts[i];
+
+        char letter = option_to_ascii(opt->type);
+        options[offset++] = letter;
+        options_ordered[ordered_offset++] = letter;
+
+        switch (opt->type) {
+            case OSFP_TCP_OPT_EOL:
+            case OSFP_TCP_OPT_NOP:
+                break;
+            case OSFP_TCP_OPT_MSS:
+                if (opt->len != OSFP_TCP_OPT_MSS_LEN) {
+                    break;
+                }
+                tcp_mss = (unsigned int)ntohs(*(unsigned short *)opt->data);
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_MSS, &tcp_mss, sizeof(tcp_mss));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_mss);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_WSCALE:
+                if (opt->len != OSFP_TCP_OPT_WS_LEN) {
+                    break;
+                }
+                tcp_ws = (unsigned int)*(unsigned char *)opt->data;
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SCALING, &tcp_ws, sizeof(tcp_ws));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ws);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_TIMESTAMP:
+                if (opt->len != OSFP_TCP_OPT_TS_LEN) {
+                    break;
+                }
+                tcp_ts = !!ntohl(*(unsigned int *)(opt->data));
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP, &tcp_ts, sizeof(tcp_ts));
+                tcp_ter = !!ntohl(*(unsigned int *)(opt->data + 4));
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY, &tcp_ter, sizeof(tcp_ter));
+                ret = snprintf(options + offset, sizeof(options), "%u", tcp_ter);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+                break;
+            case OSFP_TCP_OPT_SACKOK:
+            case OSFP_TCP_OPT_SACK:
+            case OSFP_TCP_OPT_ECHO:
+            case OSFP_TCP_OPT_ECHOREPLY:
+            case OSFP_TCP_OPT_POCONN:
+            case OSFP_TCP_OPT_POSVC:
+                break;
+            default:
+                ret = snprintf(options + offset, sizeof(options), "%u", opt->type);
+                if (ret < 0 || offset + ret > maxoffset) {
+                    break;
+                }
+                offset += ret;
+        }
+
+        options[offset++] = ',';
+        options[offset] = 0;
+        options_ordered[ordered_offset] = 0;
+    }
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OPTIONS, options, strlen(options) + 1);
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED, options_ordered, strlen(options_ordered) + 1);
+
    return 0;
 exit:
-    return ret;
+    return -1;
+}
+
+static int osfp_fingerprinting_tcp(struct tcphdr *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp)
+{
+    unsigned int tcp_off;
+    unsigned int tcp_window_size;
+    unsigned int tcp_flags;
+
+    if (tcph == NULL || tcph_len > OSFP_TCP_DATA_OFF_MAX || fp == NULL) {
+        goto exit;
+    }
+
+    tcp_off = tcph->doff << 2;
+    tcp_window_size = ntohs(tcph->window);
+    tcp_flags = *((unsigned char *)&tcph->ack_seq + 5);
+
+    if (tcp_off != tcph_len) {
+        goto exit;
+    }
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_OFF, &tcp_off, sizeof(tcp_off));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SIZE, &tcp_window_size, sizeof(tcp_window_size));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_FLAGS, &tcp_flags, sizeof(tcp_flags));
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_WINDOW_SCALING);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_MSS);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_OPTIONS);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_TCP_OPTIONS_ORDERED);
+
+    // tcp options
+    if (tcp_off > OSFP_TCP_HEADER_LEN) {
+        osfp_fingerprinting_tcp_option((unsigned char *)tcph + OSFP_TCP_HEADER_LEN, tcp_off - OSFP_TCP_HEADER_LEN, fp);
+    }
+
+    return 0;
+exit:
+    return -1;
+}
+
+static int osfp_fingerprinting_ipv4(struct iphdr *iph, struct osfp_fingerprint *fp)
+{
+    if (iph == NULL || fp == NULL) {
+        goto exit;
+    }
+
+    unsigned int ip_id = !!iph->id;
+    unsigned int ip_tos = iph->tos;
+    unsigned int ip_total_length = ntohs(iph->tot_len);
+    unsigned int ip_ttl = compute_ip_ttl(iph->ttl);
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_ID, &ip_id, sizeof(ip_id));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOS, &ip_tos, sizeof(ip_tos));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
+
+    return 0;
+exit:
+    return -1;
+}
+
+static int osfp_fingerprinting_ipv6(struct ip6_hdr *iph, struct osfp_fingerprint *fp)
+{
+    if (iph == NULL || fp == NULL) {
+        goto exit;
+    }
+
+    //unsigned int ip_id = 0;
+    //unsigned int ip_tos = 0;
+    unsigned int ip_total_length = OSFP_IPV6_HEADER_LEN + ntohs(iph->ip6_ctlun.ip6_un1.ip6_un1_plen);
+    unsigned int ip_ttl = compute_ip_ttl(iph->ip6_ctlun.ip6_un1.ip6_un1_hlim);
+
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TOTAL_LENGTH, &ip_total_length, sizeof(ip_total_length));
+    OSFP_FP_SET_FIELD(fp, OSFP_FIELD_IP_TTL, &ip_ttl, sizeof(ip_ttl));
+
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_IP_ID);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_IP_TOS);
+
+    return 0;
+exit:
+    return -1;
 }

 int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version)
@@ -457,7 +581,7 @@ int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tc
        goto exit;
    }

-    osfp_fingerprint_setup_field(fp, OSFP_FIELD_OS, OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME, strlen(OSFP_FINGERPRINT_DEFAULT_OS_CLASS_NAME) + 1);
+    OSFP_FP_INIT_FIELD(fp, OSFP_FIELD_OS);

    return 0;
 exit:
@@ -481,7 +605,7 @@ int test_osfp_fingerprinting_ipv4(void)
    };

    char str_buf[2048] = "";
-    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":\"OSFP_UNKNOWN\"}";
+    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":null}";
    struct osfp_fingerprint fp = {0};

    ret = osfp_fingerprinting(iph, tcph, 32, &fp, 4);
@@ -494,6 +618,7 @@ int test_osfp_fingerprinting_ipv4(void)
        goto exit;
    }

+    ret = -1;
    if (0 != memcmp(str_buf, target, strlen(target))) {
        goto exit;
    }
@@ -520,7 +645,7 @@ int test_osfp_fingerprinting_ipv6(void)
    };

    char str_buf[2048] = "";
-    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":\"OSFP_UNKNOWN\"}";
+    const char *target = "{\"ip_id\":1,\"ip_tos\":0,\"ip_total_length\":52,\"ip_ttl\":128,\"tcp_off\":32,\"tcp_timestamp\":null,\"tcp_timestamp_echo_reply\":null,\"tcp_window_scaling\":8,\"tcp_window_size\":8192,\"tcp_flags\":2,\"tcp_mss\":1260,\"tcp_options\":\"M1260,N,W8,N,N,S,\",\"tcp_options_ordered\":\"MNWNNS\",\"os\":null}";
    struct osfp_fingerprint fp = {0};

    ret = osfp_fingerprinting(iph, tcph, 32, &fp, 4);
@@ -533,6 +658,7 @@ int test_osfp_fingerprinting_ipv6(void)
        goto exit;
    }

+    ret = -1;
    if (0 != memcmp(str_buf, target, strlen(target))) {
        goto exit;
    }
@@ -561,6 +687,7 @@ int test_osfp_fingerprinting_tcp_option(void)
        goto exit;
    }

+    ret = -1;
    if (fp.fields[OSFP_FIELD_TCP_OPTIONS].value_len != strlen(target_options) + 1)
    {
        goto exit;
--- a/src/osfp_fingerprint.h
+++ b/src/osfp_fingerprint.h
@@ -65,6 +65,7 @@ static inline unsigned int osfp_fingerprint_get_field_type(enum osfp_field_id fi
    return fp_fields[field_id].type;
 }

+struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root);
 int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str);
 int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format);
 int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version);
--- a/src/osfp_score_db.c
+++ b/src/osfp_score_db.c
@@ -1,6 +1,6 @@
 #include "osfp_common.h"

-#include "MESA_osfp.h"
+#include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_score_db.h"
 #include "osfp_log.h"
@@ -251,6 +251,30 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
        goto exit;
    }

+    // prefileter
+    struct osfp_prefilter_hash_element *element = NULL;
+    struct osfp_fingerprint *fp = osfp_fingerprint_from_cjson(entry);
+    if (fp == NULL) {
+        goto exit;
+    }
+    HASH_FIND(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    if (element == NULL) {
+        element = (struct osfp_prefilter_hash_element *)calloc(1, sizeof(struct osfp_prefilter_hash_element));
+        if (element == NULL) {
+            free(fp);
+            goto exit;
+        }
+
+        element->fp_json = cJSON_Print(entry);
+        element->fp = fp;
+        element->os_class = os_class;
+        HASH_ADD_KEYPTR(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    } else {
+        // TODO: same fingerprints with different os should not insert into prefilter hash table, now just tag
+        element->repeated++;
+    }
+
+    // field score db
    for (i = 0; i < OSFP_FIELD_OS; i++) {
        db = &score_db->field_score_dbs[i];
        if (db == NULL) {
@@ -373,7 +397,7 @@ int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file)

    for (i = 0; i < OSFP_FIELD_MAX; i++) {
        field_score_db = &score_db->field_score_dbs[i];
-        if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS) {
+        if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS_ORDERED) {
            score_db->perfect_score += osfp_fingerprint_get_field_importance(i);
        }
    }
@@ -391,6 +415,42 @@ exit:
    return ret;
 }

+int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
+{
+    int ret, i;
+    unsigned int value_buffer_used = 0;
+    char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX];
+    struct osfp_prefilter_hash_element *element = NULL;
+
+    if (score_db->prefilter_head == NULL) {
+        return 0;
+    }
+
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            continue;
+        }
+        if (fp->fields[i].value && fp->fields[i].value_len != 0) {
+            memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len);
+            value_buffer_used += fp->fields[i].value_len;
+        }
+    }
+
+    HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element);
+    if (element == NULL) {
+        return 0;
+    }
+
+    if (element->repeated) {
+        return 0;
+    }
+
+    memset(result_score, 0, sizeof(struct osfp_os_class_score));
+    result_score->scores[element->os_class] = OSFP_PERCENTILE;
+
+    return 1;
+}
+
 int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
 {
    int ret = OSFP_EINVAL, i, j;
@@ -410,6 +470,7 @@ int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, stru
        goto exit;
    }

+    // score
    memset(result_score, 0, sizeof(struct osfp_os_class_score));

    perfect_score = score_db->perfect_score;
@@ -447,6 +508,11 @@ int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, stru
                result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * tmp_score) / entry_count;
            }
        }
+
+        if (i == OSFP_FIELD_TCP_OPTIONS) {
+            // if OSFP_FIELD_TCP_OPTIONS matched OSFP_FIELD_TCP_OPTIONS_ORDERED is not needed
+            i++;
+        }
    }

    return OSFP_NOERR;
@@ -534,8 +600,27 @@ void osfp_score_db_destroy(struct osfp_score_db *score_db)
 {
    int i;
    struct osfp_field_score_db *db;
+    struct osfp_prefilter_hash_element *element = NULL;
+    struct osfp_prefilter_hash_element *tmp = NULL;

    if (score_db) {
+        // prefilter
+        if (score_db->prefilter_head) {
+            HASH_ITER(hh, score_db->prefilter_head, element, tmp) {
+                HASH_DELETE(hh, score_db->prefilter_head, element);
+                if (element) {
+                    if (element->fp) {
+                        free(element->fp);
+                    }
+                    if (element->fp_json) {
+                        free(element->fp_json);
+                    }
+                    free(element);
+                }
+            }
+        }
+
+        // field score db
        for (i = 0; i < OSFP_FIELD_MAX; i++) {
            db = &score_db->field_score_dbs[i];
            if (db->destroy && db->data) {
--- a/src/osfp_score_db.h
+++ b/src/osfp_score_db.h
@@ -1,6 +1,7 @@
 #ifndef __OSFP_SCORE_DB_H__
 #define __OSFP_SCORE_DB_H__

+#include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_common.h"

@@ -21,8 +22,20 @@ struct osfp_field_score_db {
    struct osfp_os_class_score *(*match)(void *, void *, unsigned int);
 };

+struct osfp_prefilter_hash_element {
+    struct osfp_fingerprint *fp;
+    char *fp_json;
+    unsigned int os_class;
+
+    unsigned int repeated;
+    UT_hash_handle hh;
+};
+
 struct osfp_score_db {
    unsigned int entry_count;
+
+    struct osfp_prefilter_hash_element *prefilter_head;
+
    unsigned int perfect_score;
    unsigned int os_class_entry_count[OSFP_OS_CLASS_MAX];
    struct osfp_field_score_db field_score_dbs[OSFP_FIELD_MAX];
@@ -32,6 +45,7 @@ char *osfp_score_db_read_file(char *fp_file);
 void osfp_score_db_debug_print(struct osfp_score_db *score_db);

 int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file);
+int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);
 int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score);

 struct osfp_score_db *osfp_score_db_create(void);
--- a/src/version.map
+++ b/src/version.map
@@ -1,4 +1,4 @@
 {
-  global: MESA_osfp*;GIT_VERSION_*;
+  global: osfp*;GIT_VERSION_*;
  local: *;
 };
--- a/test/test.c
+++ b/test/test.c
@@ -6,7 +6,7 @@

 #include "cJSON.h"

-#include "MESA_osfp.h"
+#include "osfp.h"
 #include "osfp_fingerprint.h"
 #include "osfp_score_db.h"
 #include "osfp_log.h"
@@ -18,6 +18,9 @@
 #define TEST_FILE_PATH "./test.json"
 #define LOG_FILE_PATH "./osfp_test.log"

+#define OSFP_OS_CLASS_MERGED_MAX (OSFP_OS_CLASS_MAX - 2)
+#define EntryWidth 8
+
 unsigned char *data_file_path = DATA_FILE_PATH;
 unsigned char *db_file_path;
 unsigned char *test_file_path;
@@ -29,6 +32,101 @@ FILE *log_file_ptr;

 unsigned int debug_enable;

+//enum osfp_os_class_id {
+//    OSFP_OS_CLASS_UNKNOWN,      // 未知
+//    OSFP_OS_CLASS_WINDOWS,      // Windows
+//    OSFP_OS_CLASS_LINUX,        // Linux
+//    OSFP_OS_CLASS_MAC_OS,       // Mac OS
+//    OSFP_OS_CLASS_IOS,          // iOS
+//    OSFP_OS_CLASS_ANDROID,      // Android
+//    OSFP_OS_CLASS_OTHERS,       // 其他
+//    OSFP_OS_CLASS_MAX,
+//};
+// merged classes: unknown 0 windows-like 1 unix-like 2 apple-like 3 others 4
+
+unsigned int testresult[OSFP_OS_CLASS_MAX][OSFP_OS_CLASS_MAX] = {0};
+unsigned int testresult_merged[OSFP_OS_CLASS_MERGED_MAX][OSFP_OS_CLASS_MERGED_MAX] = {0};
+
+static const char *class_to_merged_name(unsigned int class)
+{
+    switch (class) {
+        case 0:
+            return "Unknown";
+        case 1:
+            return "Windows-Like";
+        case 2:
+            return "Unix-Like";
+        case 3:
+            return "Apple-Like";
+        case 4:
+            return "Others";
+    }
+    return NULL;
+}
+static unsigned int class_to_merged_class(unsigned int class)
+{
+    switch (class) {
+        case OSFP_OS_CLASS_UNKNOWN:
+            return 0;
+        case OSFP_OS_CLASS_WINDOWS:
+            return 1;
+        case OSFP_OS_CLASS_LINUX:
+            return 2;
+        case OSFP_OS_CLASS_MAC_OS:
+            return 3;
+        case OSFP_OS_CLASS_IOS:
+            return 3;
+        case OSFP_OS_CLASS_ANDROID:
+            return 2;
+        case OSFP_OS_CLASS_OTHERS:
+            return 4;
+    }
+    return 0;
+}
+
+static void testresult_class_merge()
+{
+    int i,j;
+    for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+        for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
+            testresult_merged[class_to_merged_class(i)][class_to_merged_class(j)] += testresult[i][j];
+        }
+    }
+}
+
+static void print_confusion_matrix(unsigned int *result, unsigned int os_class_max, const char *(*get_name)(unsigned int))
+{
+    int i,j;
+    int matched = 0, missed = 0;
+
+    for (i = 0; i < os_class_max; i++) {
+        printf("%*s(%c)", EntryWidth-3, " ", 'a' + i);
+    }
+    printf("    <-" " classified as" "\n");
+
+    for (i = 0; i < os_class_max; i++) {
+        printf("%*.*s", EntryWidth, EntryWidth-2, "----------");
+    }
+
+    printf("\n");
+
+
+    for (i = 0; i < os_class_max; i++) {
+        for (j = 0; j < os_class_max; j++) {
+            if (i == j) {
+                matched += *(result + os_class_max * i + j);
+            } else {
+                missed += *(result + os_class_max * i + j);
+            }
+            printf(" %*d", EntryWidth-1, *(result + os_class_max * i + j));
+        }
+        printf("    (%c): " "class" " %s\n", 'a' + i, get_name(i));
+    }
+
+    printf("miss rate: %u%%\n", 100 * missed / (matched + missed));
+}
+
+
 void test_data_prepare()
 {
    char *file_buffer;
@@ -83,6 +181,8 @@ void test_miss_rate()
    unsigned int other_count = 0;
    unsigned int unknown_count = 0;
    unsigned int identify_failed_count = 0;
+    unsigned int prefiltered_count = 0;
+    unsigned int prefiltered_wrong_count = 0;
    unsigned int wrong_count = 0;
    unsigned int verified_count = 0;
    unsigned int fingerprint_count = 0;
@@ -105,7 +205,7 @@ void test_miss_rate()
        osfp_log_level_set(OSFP_LOG_LEVEL_DEBUG);
    }

-    struct osfp_db *osfp_db = MESA_osfp_db_new(db_file_path);
+    struct osfp_db *osfp_db = osfp_db_new(db_file_path);
    if (osfp_db == NULL) {
        printf("could not create osfp context. fingerprints file: %s\n", db_file_path);
        exit(1);
@@ -123,18 +223,33 @@ void test_miss_rate()

            const char *fp_str = cJSON_PrintUnformatted(entry);

-            struct osfp_result *result = MESA_osfp_json_identify(osfp_db, fp_str);
+            struct osfp_fingerprint fp = {0};
+            osfp_fingerprint_from_json(&fp, (char*)fp_str);
+            char str_buf[2048] = "";
+            osfp_fingerprint_to_json_buf(&fp, str_buf, 2048, 0);
+            fprintf(log_file_ptr, "%s\n", str_buf);
+
+            struct osfp_result *result = osfp_json_identify(osfp_db, fp_str);
            if (result == NULL) {
                identify_failed_count++;
                continue;
            }

+            testresult[result->likely_os_class][os_class]++;
+
+            if (result->details[result->likely_os_class].score == 100) {
+                prefiltered_count++;
+            }
+
            if (os_class == result->likely_os_class) {
                verified_count++;
-                MESA_osfp_result_free(result);
+                osfp_result_free(result);
                continue;
            }

+            if (result->details[result->likely_os_class].score == 100) {
+                prefiltered_wrong_count++;
+            }
            wrong_count++;

            if (result->likely_os_class == OSFP_OS_CLASS_OTHERS) {
@@ -145,27 +260,33 @@ void test_miss_rate()
                unknown_count++;
            }

-            fprintf(log_file_ptr, "expect: %s, result: %s\n", os_class_json->valuestring, MESA_osfp_result_os_name_get(result));
+            fprintf(log_file_ptr, "expect: %s, result: %s, \n", os_class_json->valuestring, osfp_result_os_name_get(result));

-            char *result_json = MESA_osfp_result_score_detail_export(result);
+            char *result_json = osfp_result_score_detail_export(result);
            if (result_json) {
                fprintf(log_file_ptr, "%s\n", result_json);
            } else {
                fprintf(log_file_ptr, "result detail error:%p\n", result);
            }
            fflush(log_file_ptr);
-            MESA_osfp_result_free(result);
+            osfp_result_free(result);
        }
    }

-    printf("total %u, failed %u, pass %u, wrong %u, other %u, unknown %u\n",
-                fingerprint_count, identify_failed_count, verified_count, wrong_count, other_count, unknown_count);
+    printf("total %u, failed %u, pass %u, prefiltered %u (wrong: %u), wrong %u, other %u, unknown %u\n",
+                fingerprint_count, identify_failed_count, verified_count, prefiltered_count, prefiltered_wrong_count,  wrong_count, other_count, unknown_count);

-    printf("miss rate: %d%%\n", 100 - (verified_count * 100 / fingerprint_count));
+    //printf("miss rate: %d%%\n", 100 - (verified_count * 100 / fingerprint_count));
+
+    testresult_class_merge();
+
+    print_confusion_matrix((unsigned int *)testresult, OSFP_OS_CLASS_MAX, osfp_os_class_id_to_name);
+
+    print_confusion_matrix((unsigned int *)testresult_merged, OSFP_OS_CLASS_MERGED_MAX, class_to_merged_name);

    printf("details in: %s\n", LOG_FILE_PATH);

-    MESA_osfp_db_free(osfp_db);
+    osfp_db_free(osfp_db);
 }

 int main(int argc, char **argv)
Author	SHA1	Message	Date
zhuzhenjun	e80ae01d68	test: show prefilter stats	2023-10-23 14:57:38 +08:00
zhuzhenjun	50e455bf8e	prefilter: fix fields buffered order	2023-10-23 14:52:21 +08:00
zhuzhenjun	7a5b2e9f07	score_db: support hash prefilter	2023-10-23 09:54:02 +08:00
zhuzhenjun	c0cfa2e61d	compile: delete Makefile.am	2023-10-23 09:52:46 +08:00
zhuzhenjun	77a47deb92	example: print fingerprints	2023-10-23 09:52:21 +08:00
zhuzhenjun	1de666a56e	test: print confusion matrix	2023-10-17 23:37:20 +08:00
zhuzhenjun	168f931da6	fingerprint/score: bring back tcp_options_ordered	2023-10-17 23:36:10 +08:00
zhuzhenjun	e62ff39d81	api: MESA_osfp_xx -> osfp_xx header: MESA_osfp.h -> osfp.h	2023-10-13 14:08:55 +08:00