v0.0.4

README.md

@@ -2,3 +2,82 @@
 
 Libosfp is a C libaray for OS fingerprinting.
 
+## install
+```
+# osfp_example depends on libpcap
+yum install -y libpcap-devel
+# build and install to ./target
+./autogen.sh; ./configure --prefix="$(pwd)/target"; make clean; make install
+```
+
+## run example
+```
+# load the fingerprint file ./fp.json and capture on eth0, filter tcp port 8888
+./target/bin/osfp_example -f ./fp.json -i eth0 "tcp port 8888"
+# outputs like this
+# ---------------------------  SYN
+# Example ipv4 header detect: --------------------------
+# Connection info: 114.64.231.114:57570 -> 172.21.0.10:8888
+# Most likely os class: Windows
+# Details:
+# {
+#         "likely":       {
+#                 "name": "Windows",
+#                 "score":        20
+#         },
+#         "detail":       [{
+#                         "name": "Windows",
+#                         "score":        20
+#                 }, {
+#                         "name": "Linux",
+#                         "score":        10
+#                 }, {
+#                         "name": "Mac OS",
+#                         "score":        1
+#                 }, {
+#                         "name": "iOS",
+#                         "score":        0
+#                 }, {
+#                         "name": "Android",
+#                         "score":        7
+#                 }]
+# }
+```
+
+## sample
+```
+#include "stdio.h" 
+#include "osfp.h" 
+ 
+char iph[] = { 
+  0x45, 0x00, 0x00, 0x34, 0x51, 0xc4, 0x40, 0x00, 
+  0x80, 0x06, 0xe7, 0x27, 0xc0, 0xa8, 0x73, 0x08, 
+  0x6a, 0xb9, 0x23, 0x6e 
+}; 
+ 
+char tcph[] = { 
+  0xc1, 0xbd, 0x00, 0x50, 0x3d, 0x58, 0x51, 0x60, 
+  0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x20, 0x00, 
+  0x3d, 0x3a, 0x00, 0x00, 0x02, 0x04, 0x04, 0xec, 
+  0x01, 0x03, 0x03, 0x08, 0x01, 0x01, 0x04, 0x02 
+};  
+ 
+int main(int argc, char **argv) 
+{ 
+    const char *json_file_path = "./fp.json";
+ 
+    struct iphdr *l3_hdr = (struct iphdr *)iph; 
+    struct tcphdr *l4_hdr = (struct tcphdr *)tcph; 
+    size_t l4_hdr_len = sizeof(tcph); 
+ 
+    struct osfp_db *db = osfp_db_new(json_file_path); 
+    if (db) {
+        struct osfp_result *result = osfp_ipv4_identify(db, l3_hdr, l4_hdr, l4_hdr_len); 
+        if (result) { 
+            printf("likely os: %s\n", osfp_result_os_name_get(result)); 
+            printf("details: \n%s\n", osfp_result_score_detail_export(result)); 
+            osfp_db_free(db); 
+        } 
+    }
+}
+```

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([libosfp],[0.0.2],[zhuzhenjun@geedgenetworks.com])
+AC_INIT([libosfp],[0.0.4],[zhuzhenjun@geedgenetworks.com])
 AM_INIT_AUTOMAKE([foreign])
 
 #m4_ifndef([AM_SILENT_RULES], [m4_define([AM_SILENT_RULES],[])])AM_SILENT_RULES([yes])
@@ -7,7 +7,7 @@ AC_CONFIG_MACRO_DIR([m4])
 AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [enable debug info])], [enable_debug=$enableval], [enable_debug=no])
 
 AS_IF([test "x$enable_debug" = xyes],
-      [CFLAGS="-ggdb3 -O0 -fsanitize=address -fno-omit-frame-pointer"],
+      [CFLAGS="-DDEBUGLOG -ggdb3 -O0 -fsanitize=address -fno-omit-frame-pointer"],
       [CFLAGS="-g -O2"])
 
 AC_PROG_CC

example/Makefile.am

@@ -1,4 +1,4 @@
-bin_PROGRAMS = osfp_example
+bin_PROGRAMS = osfp_example sample
 
 osfp_example_SOURCES = \
         osfp_example.c
@@ -11,3 +11,13 @@ osfp_example_LDFLAGS = \
 
 osfp_example_CFLAGS = \
         -I../src
+
+
+sample_SOURCES = \
+        sample.c
+
+sample_LDADD = \
+	../src/.libs/libosfp.la
+
+sample_CFLAGS = \
+	-I../src

example/osfp_example.c

@@ -16,6 +16,7 @@
 
 #include "osfp_common.h"
 #include "osfp.h"
+#include "osfp_log.h"
 #include "osfp_fingerprint.h"
 #include "osfp_score_db.h"
 
@@ -163,6 +164,7 @@ unsigned char *fp_file_path;
 unsigned char *fp_output_file_path;
 FILE *fingerprinting_output_fp;
 
+unsigned int debug_enable;
 unsigned char *if_name;
 unsigned char *pcap_file_name;
 unsigned char *bpf_string;
@@ -466,6 +468,8 @@ void example_detect(struct osfp_db *osfp_db, Packet *p)
     printf("Details:\n");
     printf("%s\n", osfp_result_score_detail_export(result));
 
+    osfp_result_free(result);
+
 exit:
     return;
 }
@@ -483,10 +487,16 @@ void process_packet(char *user, struct pcap_pkthdr *h, u_char *pkt)
     }
 
     // only for tcp syn request packet
-    if (!p->tcph->syn || p->tcph->ack) {
+    if (!p->tcph->syn) {
         goto exit;
     }
 
+    if (p->tcph->ack) {
+        printf("---------------------------  SYN/ACK\n");
+    } else {
+        printf("---------------------------  SYN\n");
+    }
+
     if (p->iph) {
         PrintInet(AF_INET, (const void *)&(p->src.addr_data32[0]), p->srcip, sizeof(p->srcip));
         PrintInet(AF_INET, (const void *)&(p->dst.addr_data32[0]), p->dstip, sizeof(p->dstip));
@@ -508,7 +518,7 @@ int main(int argc, char *argv[])
 {
     int r;
 
-    while ((r = getopt(argc, argv, "+f:i:r:o:")) != -1) {
+    while ((r = getopt(argc, argv, "+f:i:r:o:d")) != -1) {
         switch(r) {
             case 'f':
                 if (fp_file_path) {
@@ -538,6 +548,9 @@ int main(int argc, char *argv[])
                 }
                 fp_output_file_path = (unsigned char*)optarg;
                 break;
+            case 'd':
+                debug_enable = 1;
+                break;
             default:
                 usage();
                 break;
@@ -611,13 +624,16 @@ int main(int argc, char *argv[])
         fp_file_path = DEFAULT_FP_FILE_PATH;
     }
 
+    if (debug_enable) {
+        osfp_log_level_set(OSFP_LOG_LEVEL_DEBUG);
+    }
+
     struct osfp_db *osfp_db = osfp_db_new(fp_file_path);
     if (osfp_db == NULL) {
         printf("could not create osfp context. fingerprints file: %s\n", fp_file_path);
         exit(1);
     }
 
-
     osfp_score_db_debug_print(osfp_db->score_db);
 
     // loop

example/sample.c

@@ -0,0 +1,34 @@
+#include "stdio.h" 
+#include "osfp.h" 
+ 
+char iph[] = { 
+  0x45, 0x00, 0x00, 0x34, 0x51, 0xc4, 0x40, 0x00, 
+  0x80, 0x06, 0xe7, 0x27, 0xc0, 0xa8, 0x73, 0x08, 
+  0x6a, 0xb9, 0x23, 0x6e 
+}; 
+ 
+char tcph[] = { 
+  0xc1, 0xbd, 0x00, 0x50, 0x3d, 0x58, 0x51, 0x60, 
+  0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x20, 0x00, 
+  0x3d, 0x3a, 0x00, 0x00, 0x02, 0x04, 0x04, 0xec, 
+  0x01, 0x03, 0x03, 0x08, 0x01, 0x01, 0x04, 0x02 
+};  
+ 
+int main(int argc, char **argv) 
+{ 
+    const char *json_file_path = "./fp.json";
+ 
+    struct iphdr *l3_hdr = (struct iphdr *)iph; 
+    struct tcphdr *l4_hdr = (struct tcphdr *)tcph; 
+    size_t l4_hdr_len = sizeof(tcph); 
+ 
+    struct osfp_db *db = osfp_db_new(json_file_path); 
+    if (db) {
+        struct osfp_result *result = osfp_ipv4_identify(db, l3_hdr, l4_hdr, l4_hdr_len); 
+        if (result) { 
+            printf("likely os: %s\n", osfp_result_os_name_get(result)); 
+            printf("details: \n%s\n", osfp_result_score_detail_export(result)); 
+            osfp_db_free(db); 
+        } 
+    }
+}

src/osfp.c

@@ -6,11 +6,13 @@
 #include "osfp_log.h"
 
 #define OSFP_DEFAULT_RESULT_BUFLEN_MAX  512
+#define OSFP_LOWEST_SCORE_LIMIT 20
 
 static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score)
 {
     int i;
     unsigned int tmp_score;
+    unsigned int sum_score;
     unsigned int likely_score;
     enum osfp_os_class_id likely_os_class;
     struct osfp_result *result;
@@ -22,6 +24,7 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
 
     likely_score = 0;
     likely_os_class = OSFP_OS_CLASS_OTHERS;
+    sum_score = 0;
 
     for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
         tmp_score = os_class_score->scores[i];
@@ -31,27 +34,37 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas
             likely_os_class = i;
         }
         result->detail.scores[i] = tmp_score;
+        sum_score += tmp_score;
+    }
+
+    if (sum_score) {
+        for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
+            result->detail.possibility[i] = OSFP_PERCENTILE * result->detail.scores[i] / sum_score;
+        }
     }
 
     result->likely_score = likely_score;
     result->likely_os_class = likely_os_class;
+    result->likely_possibility = result->detail.possibility[likely_os_class];
 
-    if (likely_score < 10) {
+    if (likely_score < OSFP_LOWEST_SCORE_LIMIT) {
         result->likely_os_class = OSFP_OS_CLASS_OTHERS;
         result->likely_score = 0;
+        result->likely_possibility = 0;
     } else {
         for (i = 0; i < OSFP_OS_CLASS_MAX; i++) {
             if (likely_os_class == i) {
                 continue;
             }
             if (likely_score == os_class_score->scores[i]) {
-                if (likely_os_class == OSFP_OS_CLASS_LINUX && os_class_score->scores[i] == OSFP_OS_CLASS_ANDROID) {
+                if (likely_os_class == OSFP_OS_CLASS_LINUX && i == OSFP_OS_CLASS_ANDROID) {
                     continue;
-                } else if (likely_os_class == OSFP_OS_CLASS_MAC_OS && os_class_score->scores[i] == OSFP_OS_CLASS_IOS) {
+                } else if (likely_os_class == OSFP_OS_CLASS_MAC_OS && i == OSFP_OS_CLASS_IOS) {
                     continue;
                 } else {
                     result->likely_os_class = OSFP_OS_CLASS_UNKNOWN;
                     result->likely_score = 0;
+                    result->likely_possibility = 0;
                     break;
                 }
             }
@@ -103,6 +116,7 @@ char *osfp_result_score_detail_export(struct osfp_result *result)
     if (os_score) {
         cJSON_AddStringToObject(os_score, "name", osfp_os_class_id_to_name(result->likely_os_class));
         cJSON_AddNumberToObject(os_score, "score", result->likely_score);
+        //cJSON_AddNumberToObject(os_score, "possibility", result->likely_possibility);
     }
 
     array = cJSON_AddArrayToObject(root, "detail");
@@ -112,6 +126,7 @@ char *osfp_result_score_detail_export(struct osfp_result *result)
             if (os_score) {
                 cJSON_AddStringToObject(os_score, "name", osfp_os_class_id_to_name(i));
                 cJSON_AddNumberToObject(os_score, "score", result->detail.scores[i]);
+                //cJSON_AddNumberToObject(os_score, "possibility", result->detail.possibility[i]);
                 cJSON_AddItemToArray(array, os_score);
             }
         }

src/osfp.h

@@ -27,8 +27,9 @@ enum osfp_os_class_id {
     OSFP_OS_CLASS_MAX,
 };
 
-struct osfp_os_class_score {
+struct osfp_os_result_detail {
     unsigned int scores[OSFP_OS_CLASS_MAX];
+    unsigned int possibility[OSFP_OS_CLASS_MAX];
 };
 
 struct osfp_result {
@@ -36,8 +37,9 @@ struct osfp_result {
 
     enum osfp_os_class_id   likely_os_class;
     unsigned int            likely_score;
+    unsigned int            likely_possibility;
 
-    struct osfp_os_class_score detail;
+    struct osfp_os_result_detail detail;
 };
 
 struct osfp_db {

src/osfp_common.h

@@ -139,6 +139,7 @@ enum osfp_error_code {
 #define OSFP_OS_CLASS_FLAG_IOS       OSFP_BIT_U32(OSFP_OS_CLASS_IOS)
 #define OSFP_OS_CLASS_FLAG_ANDROID   OSFP_BIT_U32(OSFP_OS_CLASS_ANDROID)
 
+#define OSFP_PERCENTILE 100
 
 const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class);
 enum osfp_os_class_id osfp_os_class_name_to_id(char *name);

src/osfp_log.c

@@ -4,7 +4,7 @@
 /* The maximum length of the log message */
 #define OSFP_MAX_LOG_MSG_LEN 2048
 
-unsigned int osfp_log_level = OSFP_LOG_LEVEL_INFO;
+unsigned int g_osfp_log_level = OSFP_LOG_LEVEL_WARNING;
 
 void osfp_log_message(unsigned int x, const char *file, const int line, const char *func, const char *msg)
 {
@@ -32,11 +32,15 @@ void osfp_log_message(unsigned int x, const char *file, const int line, const ch
             break;
     }
 
+    if (fprintf(stdout, "%s\n", buffer) < 0) {
+        printf("Error writing to stream using fprintf\n");
+    }
+    fflush(stdout);
 }
 
 void osfp_log(unsigned int x, const char *file, const int line, const char *func, const char *fmt, ...)
 {
-    if (osfp_log_level >= x ) {
+    if (g_osfp_log_level <= x ) {
         char msg[OSFP_MAX_LOG_MSG_LEN];
         va_list ap;
         va_start(ap, fmt);
@@ -46,8 +50,8 @@ void osfp_log(unsigned int x, const char *file, const int line, const char *func
     }
 }
 
-void osfp_log_level_set(osfp_log_level_t level)
+void osfp_log_level_set(enum osfp_log_level level)
 {
-    osfp_log_level = level;
+    g_osfp_log_level = level;
 }
 

src/osfp_log.h

@@ -1,14 +1,14 @@
 #ifndef __OSFP_LOG_H__
 #define __OSFP_LOG_H__
 
-typedef enum osfp_log_level {
+enum osfp_log_level {
     OSFP_LOG_LEVEL_DEBUG,
     OSFP_LOG_LEVEL_INFO,
     OSFP_LOG_LEVEL_WARNING,
     OSFP_LOG_LEVEL_ERROR
-} osfp_log_level_t;
+};
 
-#ifndef DEBUG
+#ifndef DEBUGLOG
 #define osfp_log_debug(...) do { } while (0)
 #else
 #define osfp_log_debug(...) osfp_log(OSFP_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __FUNCTION__, __VA_ARGS__)
@@ -18,7 +18,7 @@ typedef enum osfp_log_level {
 #define osfp_log_error(...) osfp_log(OSFP_LOG_LEVEL_ERROR, __FILE__, __LINE__,  __FUNCTION__,__VA_ARGS__)
 
 
-void osfp_log_level_set(osfp_log_level_t level);
+void osfp_log_level_set(enum osfp_log_level level);
 void osfp_log(unsigned int x, const char *file, const int line, const char *func, const char *fmt, ...);
 
 #endif

src/osfp_score_db.c

@@ -5,7 +5,8 @@
 #include "osfp_score_db.h"
 #include "osfp_log.h"
 
-#define OSFP_PERCENTILE 100
+#define PERFECT_SCORE_EXPECTED_RATE 0.5f
+#define FIELD_VALUE_DUP_RATE_MAX 0.5f
 
 #define OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX 65536
 
@@ -209,30 +210,31 @@ char *osfp_score_db_read_file(char *fp_file)
     struct stat st;
 
     if (0 > stat(fp_file, &st)) {
-        printf("stat() on '%s' failed.\n", fp_file);
+        osfp_log_error("stat() failed on '%s'.\n", fp_file);
         goto exit;
     }
 
     if (st.st_size == 0) { 
-        printf("Empty file: %s.\n", fp_file);
+        osfp_log_error("Empty file: '%s'.\n", fp_file);
         goto exit;
     }
 
     file_len = (unsigned int)st.st_size;
     file_buffer = malloc(file_len + 1);
     if (file_buffer == NULL) {
-        printf("Not enough memory for file buffer. file name: %s\n",fp_file);
+        osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file);
         goto exit;
     }
 
     fp = fopen(fp_file, "r");
     if (fp == NULL) {
-        printf("Cannot open '%s' for reading.\n", fp_file);
+        osfp_log_error("Cannot open '%s' for reading.\n", fp_file);
         goto exit;
     }
 
     ret = fread(file_buffer, 1, file_len, fp);
     if (ret < 0) {
+        osfp_log_error("fread() failed on '%s'.\n", fp_file);
         free(file_buffer);
         fclose(fp);
         goto exit;
@@ -318,14 +320,14 @@ int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry)
 
         field = cJSON_GetObjectItem(entry, osfp_fingerprint_get_field_name(i));
         if (field == NULL) {
-            printf("json entry missing field: %s\n%s\n",
+            osfp_log_info("json entry missing field: %s\n%s\n",
                         osfp_fingerprint_get_field_name(i), cJSON_Print(entry));
             continue;
         }
 
         ret = osfp_score_db_load_field(db, field, os_class);
         if (ret != 0) {
-            printf("json entry field load failed. field: %s\n%s\n",
+            osfp_log_info("json entry field load failed. field: %s\n%s\n",
                         osfp_fingerprint_get_field_name(i), cJSON_Print(entry));
             continue;
         }
@@ -353,12 +355,14 @@ int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file)
 
     file_buffer = osfp_score_db_read_file(fp_file);
     if (file_buffer == NULL) {
+        osfp_log_error("read file: '%s'\n", fp_file);
         ret = OSFP_ERR_SCORE_DB_READ_FILE;
         goto exit;
     }
 
     root = cJSON_Parse(file_buffer);
     if (root == NULL) {
+        osfp_log_error("parse json: '%s'\n", fp_file);
         ret = OSFP_ERR_SCORE_DB_PARSE_FILE;
         goto exit;
     }
@@ -378,11 +382,13 @@ int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file)
 
     for (i = 0; i < OSFP_FIELD_MAX; i++) {
         field_score_db = &score_db->field_score_dbs[i];
-        if (field_score_db->enabled) {
+        if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS) {
             score_db->perfect_score += osfp_fingerprint_get_field_importance(i);
         }
     }
 
+    score_db->perfect_score = score_db->perfect_score * PERFECT_SCORE_EXPECTED_RATE;
+
     ret = OSFP_NOERR;
 exit:
     if (root) {
@@ -441,18 +447,15 @@ int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, stru
         for (j = 0; j < OSFP_OS_CLASS_MAX; j++) {
             entry_count = score_db->os_class_entry_count[j];
             tmp_score = os_class_score_matched->scores[j];
+            tmp_score = tmp_score < (entry_count * FIELD_VALUE_DUP_RATE_MAX) ? tmp_score : (entry_count * FIELD_VALUE_DUP_RATE_MAX);
             if (entry_count == 0 || tmp_score == 0) {
                 continue;
             }
             if (0 == flags || flags & OSFP_BIT_U32(j)) {
-                printf("%s %s: ((%d * %u / %u) * %u ) / %u\n", osfp_fingerprint_get_field_name(i), osfp_os_class_id_to_name(j), OSFP_PERCENTILE, importance, perfect_score, os_class_score_matched->scores[j], entry_count);
+                osfp_log_debug("%s %s: ((%d * %u / %u) * %u ) / %u\n", osfp_fingerprint_get_field_name(i), osfp_os_class_id_to_name(j), OSFP_PERCENTILE, importance, perfect_score, os_class_score_matched->scores[j], entry_count);
-                result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * os_class_score_matched->scores[j]) / entry_count;
+                result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * tmp_score) / entry_count;
             }
         }
-
-        if (i == OSFP_FIELD_TCP_OPTIONS) {
-            i++;
-        }
     }
 
     return OSFP_NOERR;
@@ -499,7 +502,7 @@ struct osfp_score_db *osfp_score_db_create(void)
 
         db->enabled = osfp_fingerprint_get_field_enabled(i);
         if (!db->enabled) {
-            osfp_log_warning("field disabled: %s", "");
+            osfp_log_info("fingerprint field disabled: %s", osfp_fingerprint_get_field_name(i));
             continue;
         }
 
@@ -518,13 +521,13 @@ struct osfp_score_db *osfp_score_db_create(void)
                 db->match = osfp_score_db_hash_match;
                 break;
             default:
-                osfp_log_debug("unsupported type: %u", db->type);
+                osfp_log_debug("fingerprint field unsupported type: %u", db->type);
                 goto exit;
         }
 
         db->data = db->create();
         if (db->data == NULL) {
-            osfp_log_debug("create failed. field: %s", osfp_fingerprint_get_field_name(i));
+            osfp_log_debug("field db create failed. field: %s", osfp_fingerprint_get_field_name(i));
             goto exit;
         }
     }
@@ -545,8 +548,10 @@ void osfp_score_db_destroy(struct osfp_score_db *score_db)
     if (score_db) {
         for (i = 0; i < OSFP_FIELD_MAX; i++) {
             db = &score_db->field_score_dbs[i];
-            db->destroy(db->data);
+            if (db->destroy && db->data) {
-            db->data = NULL;
+                db->destroy(db->data);
+                db->data = NULL;
+            }
         }
         free(score_db);
     }

src/osfp_score_db.h

@@ -4,6 +4,10 @@
 #include "osfp.h"
 #include "osfp_fingerprint.h"
 
+struct osfp_os_class_score {
+    unsigned int scores[OSFP_OS_CLASS_MAX];
+};
+
 struct osfp_field_score_db {
     unsigned int    enabled;
     unsigned int    type;