diff --git a/src/osfp.c b/src/osfp.c index 5b933c5..53fa5a5 100644 --- a/src/osfp.c +++ b/src/osfp.c @@ -5,14 +5,12 @@ #include "osfp_score_db.h" #include "osfp_log.h" -#define OSFP_DEFAULT_RESULT_BUFLEN_MAX 512 #define OSFP_LOWEST_SCORE_LIMIT 20 -static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score) +static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_class_score, const char *matched) { int i; unsigned int tmp_score; - unsigned int sum_score; unsigned int likely_score; enum osfp_os_class_id likely_os_class; struct osfp_result *result; @@ -24,8 +22,8 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas likely_score = 0; likely_os_class = OSFP_OS_CLASS_OTHERS; - sum_score = 0; + // likely os score for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { tmp_score = os_class_score->scores[i]; @@ -34,20 +32,16 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas likely_os_class = i; } result->details[i].score = tmp_score; - sum_score += tmp_score; - } - - if (sum_score) { - for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { - result->details[i].possibility = OSFP_PERCENTILE * result->details[i].score / sum_score; - } } if (likely_score == OSFP_PERCENTILE) { + // prefiltered ; } else if (likely_score < OSFP_LOWEST_SCORE_LIMIT) { + // too low to tell os class likely_os_class = OSFP_OS_CLASS_OTHERS; } else { + // when the tied likely scores appear between win/apple-like/unix-like, we throw unknown for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { if (likely_os_class == i) { continue; @@ -66,7 +60,7 @@ static struct osfp_result *osfp_result_build(struct osfp_os_class_score *os_clas } result->likely_os_class = likely_os_class; - + result->matched = matched; return result; exit: return NULL; @@ -91,22 +85,23 @@ const char *osfp_result_os_name_get(struct osfp_result *result) char *osfp_result_score_detail_export(struct osfp_result *result) { int i; - char *result_str = NULL; - cJSON *root = NULL; + char *result_str; + cJSON *root; cJSON *array; cJSON *os_score; + cJSON *matched; osfp_profile_cycle(c1); osfp_profile_cycle(c2); + osfp_profile_get_cycle(c1); if (result == NULL) { - goto exit; + return NULL; } if (result->json_str != NULL) { - result_str = result->json_str; - goto exit; + return result->json_str; } root = cJSON_CreateObject(); @@ -134,6 +129,13 @@ char *osfp_result_score_detail_export(struct osfp_result *result) } } + if (result->matched) { + matched = cJSON_AddObjectToObject(root, "matched"); + if (matched) { + cJSON_AddStringToObject(matched, "fingerprint", result->matched); + } + } + result_str = cJSON_Print(root); if (result_str == NULL) { goto exit; @@ -141,15 +143,15 @@ char *osfp_result_score_detail_export(struct osfp_result *result) result->json_str = result_str; + osfp_profile_get_cycle(c2); + osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1); + + return result_str; exit: if (root) { cJSON_Delete(root); } - if (result_str) { - osfp_profile_get_cycle(c2); - osfp_profile_counter_update(&osfp_profile_result_export, c2 - c1); - } - return result_str; + return NULL; } void osfp_result_free(struct osfp_result *result) @@ -164,15 +166,16 @@ void osfp_result_free(struct osfp_result *result) struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len) { - int ret = OSFP_EINVAL; + int ret; struct osfp_fingerprint fp; struct osfp_os_class_score os_class_score; struct osfp_result *result; + const char *matched; osfp_profile_cycle(c1); osfp_profile_cycle(c2); - if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr == 0) { + if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr_len == 0) { goto exit; } @@ -185,10 +188,10 @@ struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, } osfp_profile_get_cycle(c1); - ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score); + matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score); osfp_profile_get_cycle(c2); osfp_profile_counter_update(&osfp_profile_prefilter, c2 - c1); - if (ret <= 0) { + if (matched == NULL) { osfp_profile_get_cycle(c1); ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score); osfp_profile_get_cycle(c2); @@ -199,7 +202,7 @@ struct osfp_result *osfp_ipv4_identify(struct osfp_db *db, struct iphdr* l3_hdr, } osfp_profile_get_cycle(c1); - result = osfp_result_build(&os_class_score); + result = osfp_result_build(&os_class_score, matched); osfp_profile_get_cycle(c2); osfp_profile_counter_update(&osfp_profile_result_build, c2 - c1); if (result == NULL) { @@ -213,26 +216,45 @@ exit: struct osfp_result *osfp_ipv6_identify(struct osfp_db *db, struct ip6_hdr* l3_hdr, struct tcphdr *l4_hdr, size_t l4_hdr_len) { - int ret = OSFP_EINVAL; + int ret; struct osfp_fingerprint fp; struct osfp_os_class_score os_class_score; struct osfp_result *result; + const char *matched; + + osfp_profile_cycle(c1); + osfp_profile_cycle(c2); if (db == NULL || l3_hdr == NULL || l4_hdr == NULL || l4_hdr_len == 0) { goto exit; } + osfp_profile_get_cycle(c1); ret = osfp_fingerprinting((unsigned char *)l3_hdr, (unsigned char *)l4_hdr, (unsigned int)l4_hdr_len, &fp, 6); + osfp_profile_get_cycle(c2); + osfp_profile_counter_update(&osfp_profile_fingerprinting, c2 - c1); if (ret != 0) { goto exit; } - ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score); - if (ret != 0) { - goto exit; + osfp_profile_get_cycle(c1); + matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score); + osfp_profile_get_cycle(c2); + osfp_profile_counter_update(&osfp_profile_prefilter, c2 - c1); + if (matched == NULL) { + osfp_profile_get_cycle(c1); + ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score); + osfp_profile_get_cycle(c2); + osfp_profile_counter_update(&osfp_profile_score, c2 - c1); + if (ret != 0) { + goto exit; + } } - result = osfp_result_build(&os_class_score); + osfp_profile_get_cycle(c1); + result = osfp_result_build(&os_class_score, matched); + osfp_profile_get_cycle(c2); + osfp_profile_counter_update(&osfp_profile_result_build, c2 - c1); if (result == NULL) { goto exit; } @@ -248,8 +270,9 @@ struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str) struct osfp_fingerprint fp; struct osfp_os_class_score os_class_score; struct osfp_result *result; + const char *matched; - if (db == NULL) { + if (db == NULL || json_str == NULL) { goto exit; } @@ -258,15 +281,15 @@ struct osfp_result *osfp_json_identify(struct osfp_db *db, const char *json_str) goto exit; } - ret = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score); - if (ret <= 0) { + matched = osfp_score_db_prefilter(db->score_db, &fp, &os_class_score); + if (matched == NULL) { ret = osfp_score_db_score(db->score_db, 0, &fp, &os_class_score); if (ret != 0) { goto exit; } } - result = osfp_result_build(&os_class_score); + result = osfp_result_build(&os_class_score, matched); if (result == NULL) { goto exit; } @@ -276,25 +299,27 @@ exit: return NULL; } -struct osfp_db *osfp_db_new(const char *db_json_file) +struct osfp_db *osfp_db_new(const char *fp_path) { int ret; struct osfp_db *db; + if (fp_path == NULL) { + goto exit; + } + + if (0 != access(fp_path, R_OK)) { + goto exit; + } + db = calloc(1, sizeof(struct osfp_db)); if (db == NULL) { goto exit; } - if (db_json_file != NULL) { - if (0 != access(db_json_file, R_OK)) { - goto exit; - } - - db->db_json_path = strdup((const char*)db_json_file); - if (db->db_json_path == NULL) { - goto exit; - } + db->db_json_path = strdup((const char*)fp_path); + if (db->db_json_path == NULL) { + goto exit; } db->score_db = (void *)osfp_score_db_create(); diff --git a/src/osfp_common.c b/src/osfp_common.c index 8ac9ac3..315fd88 100644 --- a/src/osfp_common.c +++ b/src/osfp_common.c @@ -1,6 +1,7 @@ #include "osfp_common.h" #include "osfp.h" +#include "osfp_log.h" unsigned int osfp_profile_enable; @@ -25,6 +26,21 @@ const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class) return osfp_os_class_name[os_class]; } +enum osfp_os_class_id osfp_os_class_name_to_id(char *name) +{ + int i, namelen; + const char *os_class_name; + + for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { + os_class_name = osfp_os_class_id_to_name(i); + if (0 == strncmp(name, os_class_name, strlen(os_class_name))) { + return (enum osfp_os_class_id)i; + } + } + + return OSFP_OS_CLASS_MAX; +} + void osfp_profile_counter_print(struct osfp_profile_counter *profile, const char *name) { printf("profile %s: avg: %lu max: %lu min: %lu curr: %lu total: %lu count: %lu\n", @@ -69,24 +85,51 @@ void osfp_profile_set(unsigned int enabled) osfp_profile_enable = enabled; } -enum osfp_os_class_id osfp_os_class_name_to_id(char *name) +char *osfp_read_file(char *fp_file) { - enum osfp_os_class_id os_class; + int ret = -1; + char *file_buffer = NULL; + unsigned int file_len = 0; + FILE *fp = NULL; + struct stat st; - if (0 == strncmp(name, OSFP_OS_CLASS_NAME_WINDOWS, strlen(OSFP_OS_CLASS_NAME_WINDOWS))) { - os_class = OSFP_OS_CLASS_WINDOWS; - } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_LINUX, strlen(OSFP_OS_CLASS_NAME_LINUX))) { - os_class = OSFP_OS_CLASS_LINUX; - } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_MAC_OS, strlen(OSFP_OS_CLASS_NAME_MAC_OS))) { - os_class = OSFP_OS_CLASS_MAC_OS; - } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_IOS, strlen(OSFP_OS_CLASS_NAME_IOS))) { - os_class = OSFP_OS_CLASS_IOS; - } else if (0 == strncmp(name, OSFP_OS_CLASS_NAME_ANDROID, strlen(OSFP_OS_CLASS_NAME_ANDROID))) { - os_class = OSFP_OS_CLASS_ANDROID; - } else { - os_class = OSFP_OS_CLASS_MAX; + if (0 > stat(fp_file, &st)) { + osfp_log_error("stat() failed on '%s'.\n", fp_file); + goto exit; } - return os_class; + if (st.st_size == 0) { + osfp_log_error("Empty file: '%s'.\n", fp_file); + goto exit; + } + + file_len = (unsigned int)st.st_size; + file_buffer = malloc(file_len + 1); + if (file_buffer == NULL) { + osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file); + goto exit; + } + + fp = fopen(fp_file, "r"); + if (fp == NULL) { + osfp_log_error("Cannot open '%s' for reading.\n", fp_file); + goto exit; + } + + ret = fread(file_buffer, 1, file_len, fp); + if (ret < 0) { + osfp_log_error("fread() failed on '%s'.\n", fp_file); + free(file_buffer); + fclose(fp); + goto exit; + } + + fclose(fp); + + file_buffer[file_len] = 0; + + return file_buffer; +exit: + return NULL; } diff --git a/src/osfp_common.h b/src/osfp_common.h index ac7b404..bb7319c 100644 --- a/src/osfp_common.h +++ b/src/osfp_common.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "utarray.h" #include "uthash.h" @@ -21,8 +22,6 @@ #include "cJSON.h" -#include "osfp.h" - static inline unsigned long long osfp_rdtsc(void) { union { @@ -186,6 +185,7 @@ struct osfp_result { char *json_str; // JSON 字符串 enum osfp_os_class_id likely_os_class; // 最可能的操作系统类别 struct osfp_result_detail details[OSFP_OS_CLASS_MAX]; // 详细结果数组 + const char *matched; // 精确匹配到的指纹特征 }; /** @@ -197,7 +197,7 @@ struct osfp_db { }; enum osfp_os_class_id osfp_os_class_name_to_id(char *name); - const char *osfp_os_class_id_to_name(enum osfp_os_class_id os_class); +char *osfp_read_file(char *fp_file); #endif diff --git a/src/osfp_fingerprint.c b/src/osfp_fingerprint.c index c57ba77..9652657 100644 --- a/src/osfp_fingerprint.c +++ b/src/osfp_fingerprint.c @@ -270,7 +270,7 @@ struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root) OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len); break; case cJSON_NULL: - //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root)); + //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root)); break; default: goto exit; @@ -352,7 +352,7 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str) OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len); break; case cJSON_NULL: - //printf("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root)); + //osfp_log_debug("fingerprint parse error: %s\n%s\n", field->string, cJSON_Print(root)); break; default: goto exit; @@ -554,7 +554,7 @@ int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tc { int ret = OSFP_EINVAL; - if (iph == NULL || tcph == NULL || fp == NULL) { + if (iph == NULL || tcph == NULL || tcph_len == 0 || fp == NULL) { goto exit; } diff --git a/src/osfp_fingerprint.h b/src/osfp_fingerprint.h index f31ed16..7a4fc88 100644 --- a/src/osfp_fingerprint.h +++ b/src/osfp_fingerprint.h @@ -68,6 +68,7 @@ static inline unsigned int osfp_fingerprint_get_field_type(enum osfp_field_id fi struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root); int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str); int osfp_fingerprint_to_json_buf(struct osfp_fingerprint *fp, char *strbuf, unsigned int buf_len, unsigned int format); + int osfp_fingerprinting(unsigned char *iph, unsigned char *tcph, unsigned int tcph_len, struct osfp_fingerprint *fp, unsigned int ip_version); int test_osfp_fingerprinting_ipv4(void); diff --git a/src/osfp_score_db.c b/src/osfp_score_db.c index 88a3bea..60df620 100644 --- a/src/osfp_score_db.c +++ b/src/osfp_score_db.c @@ -5,20 +5,29 @@ #include "osfp_score_db.h" #include "osfp_log.h" -#define PERFECT_SCORE_EXPECTED_RATE 0.5f +/* + * 一个字段的值,最多能命中单个操作系统的,指纹总数的百分比 + * 例如:linux 总指纹数量 100,其中 ip_id 字段值为 1 的指纹数为80,那么 ip_id + * 字段的得分最大为 FIELD_VALUE_DUP_RATE_MAX * 100 = 50 + */ #define FIELD_VALUE_DUP_RATE_MAX 0.5f +/* + * 由于 FIELD_VALUE_DUP_RATE_MAX 设置了单个字段得分的最大百分比,所以在归一化时,应除以这个百分比 + */ +#define PERFECT_SCORE_EXPECTED_RATE (FIELD_VALUE_DUP_RATE_MAX) + #define OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX 65536 struct osfp_score_db_array_data { - struct osfp_os_class_score *array_head; + struct osfp_field_value_count *array_head; unsigned int array_len; }; struct osfp_score_db_hash_element { char *key; unsigned int keylen; - struct osfp_os_class_score *score; + struct osfp_field_value_count *fvc; UT_hash_handle hh; }; @@ -26,13 +35,13 @@ struct osfp_score_db_hash_data { struct osfp_score_db_hash_element *hash_head; }; -static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len) +static int osfp_score_db_array_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len) { int ret = -1, i; unsigned int index; struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data; - if (array_data == NULL || os_class_score == NULL || value == NULL || len != sizeof(unsigned int)) { + if (array_data == NULL || fvc == NULL || value == NULL || len != sizeof(unsigned int)) { goto exit; } @@ -47,7 +56,7 @@ static int osfp_score_db_array_add(void *data, struct osfp_os_class_score *os_cl } for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { - array_data->array_head[index].scores[i] += os_class_score->scores[i]; + array_data->array_head[index].counts[i] += fvc->counts[i]; } return 0; @@ -55,7 +64,7 @@ exit: return ret; } -static struct osfp_os_class_score *osfp_score_db_array_match(void *data, void *value, unsigned int len) +static struct osfp_field_value_count *osfp_score_db_array_match(void *data, void *value, unsigned int len) { unsigned int index; struct osfp_score_db_array_data *array_data = (struct osfp_score_db_array_data *)data; @@ -84,7 +93,7 @@ static void *osfp_score_db_array_create(void) return NULL; } - array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_os_class_score)); + array_data->array_head = calloc(OSFP_SCORE_DB_FIELD_UINT_VALUE_MAX, sizeof(struct osfp_field_value_count)); if (array_data->array_head == NULL) { free(array_data); return NULL; @@ -106,13 +115,13 @@ static void osfp_score_db_array_destroy(void *data) { } } -static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_class_score, void *value, unsigned int len) +static int osfp_score_db_hash_add(void *data, struct osfp_field_value_count *fvc, void *value, unsigned int len) { int ret = -1, i; struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data; struct osfp_score_db_hash_element *element = NULL; - if (hash_data == NULL || os_class_score == NULL || value == NULL || len == 0) { + if (hash_data == NULL || fvc == NULL || value == NULL || len == 0) { goto exit; } @@ -124,8 +133,8 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla } element->key = strdup(value); element->keylen = len; - element->score = (struct osfp_os_class_score *)calloc(1, sizeof(struct osfp_os_class_score)); - if (element->score == NULL) { + element->fvc = (struct osfp_field_value_count *)calloc(1, sizeof(struct osfp_field_value_count)); + if (element->fvc == NULL) { free(element); element = NULL; goto exit; @@ -134,7 +143,7 @@ static int osfp_score_db_hash_add(void *data, struct osfp_os_class_score *os_cla } for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { - element->score->scores[i] += os_class_score->scores[i]; + element->fvc->counts[i] += fvc->counts[i]; } return 0; @@ -142,7 +151,7 @@ exit: return ret; } -static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *value, unsigned int len) +static struct osfp_field_value_count *osfp_score_db_hash_match(void *data, void *value, unsigned int len) { struct osfp_score_db_hash_data *hash_data = (struct osfp_score_db_hash_data *)data; struct osfp_score_db_hash_element *element = NULL; @@ -160,7 +169,7 @@ static struct osfp_os_class_score *osfp_score_db_hash_match(void *data, void *va return NULL; } - return element->score; + return element->fvc; } static void *osfp_score_db_hash_create(void) @@ -181,8 +190,8 @@ static void osfp_score_db_hash_destroy(void *data) { if (element->key) { free(element->key); } - if (element->score) { - free(element->score); + if (element->fvc) { + free(element->fvc); } free(element); } @@ -192,10 +201,136 @@ static void osfp_score_db_hash_destroy(void *data) { } } +void osfp_score_db_debug_print(struct osfp_score_db *score_db) +{ + int i; + printf("score_db:\n"); + printf("entry_count: %u\n", score_db->entry_count); + printf("total_weight: %u\n", score_db->total_weight); + + for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { + const char *name = osfp_os_class_id_to_name(i); + printf("os class %s ", name); + + printf("entry_count: %u\n", score_db->os_entry_count[i]); + printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_entry_count[i]); + } + + for (i = 0; i < OSFP_FIELD_MAX; i++) { + printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled); + printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type); + printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count); + } +} + +const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score) +{ + int ret, i; + unsigned int value_buffer_used = 0; + char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX]; + struct osfp_prefilter_hash_element *element = NULL; + + if (score_db->prefilter_head == NULL) { + return NULL; + } + + for (i = 0; i < OSFP_FIELD_OS; i++) { + if (0 == osfp_fingerprint_get_field_enabled(i)) { + continue; + } + if (fp->fields[i].value && fp->fields[i].value_len != 0) { + memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len); + value_buffer_used += fp->fields[i].value_len; + } + } + + HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element); + if (element == NULL) { + return NULL; + } + + if (element->repeated) { + return NULL; + } + + memset(result_score, 0, sizeof(struct osfp_os_class_score)); + result_score->scores[element->os_class] = OSFP_PERCENTILE; + + return (const char *)element->fp_json; +} + +int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score) +{ + int ret = -1, i, j; + + unsigned int coefficient; + unsigned int matched_count; + unsigned int os_entry_count; + unsigned int total_weight; + unsigned int field_weight; + + struct osfp_fingerprint_field *field; + struct osfp_field_value_count *fvc; // field_value_count + struct osfp_field_score_db *fdb; + + if (score_db == NULL || fp == NULL || result_score == NULL) { + goto exit; + } + + // score + memset(result_score, 0, sizeof(struct osfp_os_class_score)); + + total_weight = score_db->total_weight; + if (total_weight == 0) { + goto exit; + } + + for (i = 0; i < OSFP_FIELD_MAX; i++) { + fdb = &score_db->field_score_dbs[i]; + if (!fdb->enabled) { + continue; + } + + field = &fp->fields[i]; + if (!field->enabled) { + continue; + } + + fvc = fdb->match(fdb->data, field->value, field->value_len); + if (fvc == NULL) { + continue; + } + + coefficient = OSFP_PERCENTILE / PERFECT_SCORE_EXPECTED_RATE * score_db->field_weight[i] / total_weight; + + for (j = 0; j < OSFP_OS_CLASS_MAX; j++) { + os_entry_count = score_db->os_entry_count[j]; + matched_count = MIN(fvc->counts[j], os_entry_count * FIELD_VALUE_DUP_RATE_MAX); + + if (os_entry_count == 0 || matched_count == 0) { + continue; + } + + if (0 == flags || flags & OSFP_BIT_U32(j)) { + result_score->scores[j] += coefficient * matched_count / os_entry_count; + } + } + + // if tcp options matched tcp options ordered is not needed + if (i == OSFP_FIELD_TCP_OPTIONS) { + i++; + } + } + + return 0; +exit: + return ret; +} + static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field, enum osfp_os_class_id os_class) { int ret = -1; - struct osfp_os_class_score os_class_score = {0}; + struct osfp_field_value_count field_value_count = {0}; void *value_ptr; unsigned int value_len; @@ -216,9 +351,9 @@ static int osfp_score_db_load_field(struct osfp_field_score_db *db, cJSON *field goto exit; } - os_class_score.scores[os_class] = 1; + field_value_count.counts[os_class] = 1; - ret = db->add(db->data, &os_class_score, value_ptr, value_len); + ret = db->add(db->data, &field_value_count, value_ptr, value_len); if (ret != 0) { goto exit; } @@ -241,6 +376,7 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry goto exit; } + // get os class field = cJSON_GetObjectItem(entry, osfp_fingerprint_get_field_name(OSFP_FIELD_OS)); if (field == NULL || field->valuestring == NULL) { goto exit; @@ -301,247 +437,58 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry } score_db->entry_count++; - score_db->os_class_entry_count[os_class]++; + score_db->os_entry_count[os_class]++; return 0; exit: return ret; } -char *osfp_score_db_read_file(char *fp_file) -{ - int ret = -1; - char *file_buffer = NULL; - unsigned int file_len = 0; - FILE *fp = NULL; - struct stat st; - - if (0 > stat(fp_file, &st)) { - osfp_log_error("stat() failed on '%s'.\n", fp_file); - goto exit; - } - - if (st.st_size == 0) { - osfp_log_error("Empty file: '%s'.\n", fp_file); - goto exit; - } - - file_len = (unsigned int)st.st_size; - file_buffer = malloc(file_len + 1); - if (file_buffer == NULL) { - osfp_log_error("Not enough memory for file buffer. file name: '%s'\n",fp_file); - goto exit; - } - - fp = fopen(fp_file, "r"); - if (fp == NULL) { - osfp_log_error("Cannot open '%s' for reading.\n", fp_file); - goto exit; - } - - ret = fread(file_buffer, 1, file_len, fp); - if (ret < 0) { - osfp_log_error("fread() failed on '%s'.\n", fp_file); - free(file_buffer); - fclose(fp); - goto exit; - } - - fclose(fp); - - file_buffer[file_len] = 0; - - return file_buffer; -exit: - return NULL; -} - int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file) { - int ret = OSFP_EINVAL, i, count; + int ret = -1, i, count; char *file_buffer; - cJSON *root = NULL; cJSON *entry; + cJSON *root = NULL; struct osfp_field_score_db *field_score_db; - if (score_db == NULL) { + if (score_db == NULL || fp_file == NULL) { goto exit; } - file_buffer = osfp_score_db_read_file(fp_file); + file_buffer = osfp_read_file(fp_file); if (file_buffer == NULL) { osfp_log_error("read file: '%s'\n", fp_file); - ret = OSFP_ERR_SCORE_DB_READ_FILE; goto exit; } root = cJSON_Parse(file_buffer); if (root == NULL) { osfp_log_error("parse json: '%s'\n", fp_file); - ret = OSFP_ERR_SCORE_DB_PARSE_FILE; goto exit; } count = cJSON_GetArraySize(root); - for (i = 0; i < count; i++) { entry = cJSON_GetArrayItem(root, i); - if (entry) { - ret = osfp_score_db_load_entry(score_db, entry); - if (ret != 0) { - osfp_log_debug("json entry load failed.\n%s\n", cJSON_Print(entry)); - continue; - } + ret = osfp_score_db_load_entry(score_db, entry); + if (ret != 0) { + osfp_log_debug("json entry load failed.\n%s\n", cJSON_Print(entry)); + continue; } } - for (i = 0; i < OSFP_FIELD_MAX; i++) { - field_score_db = &score_db->field_score_dbs[i]; - if (field_score_db->enabled && i != OSFP_FIELD_TCP_OPTIONS_ORDERED) { - score_db->perfect_score += osfp_fingerprint_get_field_importance(i); - } - } - - score_db->perfect_score = score_db->perfect_score * PERFECT_SCORE_EXPECTED_RATE; - - ret = OSFP_NOERR; + ret = 0; exit: if (root) { cJSON_Delete(root); } if (file_buffer) { - free((char*)file_buffer); + free(file_buffer); } return ret; } -int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score) -{ - int ret, i; - unsigned int value_buffer_used = 0; - char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX]; - struct osfp_prefilter_hash_element *element = NULL; - - if (score_db->prefilter_head == NULL) { - return 0; - } - - for (i = 0; i < OSFP_FIELD_OS; i++) { - if (0 == osfp_fingerprint_get_field_enabled(i)) { - continue; - } - if (fp->fields[i].value && fp->fields[i].value_len != 0) { - memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len); - value_buffer_used += fp->fields[i].value_len; - } - } - - HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element); - if (element == NULL) { - return 0; - } - - if (element->repeated) { - return 0; - } - - memset(result_score, 0, sizeof(struct osfp_os_class_score)); - result_score->scores[element->os_class] = OSFP_PERCENTILE; - - return 1; -} - -int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score) -{ - int ret = OSFP_EINVAL, i, j; - - unsigned int tmp_score; - unsigned int perfect_score; - unsigned int entry_count; - unsigned int importance; - - struct osfp_os_class_score *os_class_score_matched; - enum osfp_os_class_id os_class_id; - - struct osfp_fingerprint_field *field; - struct osfp_field_score_db *field_score_db; - - if (score_db == NULL || fp == NULL || result_score == NULL) { - goto exit; - } - - // score - memset(result_score, 0, sizeof(struct osfp_os_class_score)); - - perfect_score = score_db->perfect_score; - if (perfect_score == 0) { - goto exit; - } - - for (i = 0; i < OSFP_FIELD_MAX; i++) { - field_score_db = &score_db->field_score_dbs[i]; - if (!field_score_db->enabled) { - continue; - } - - field = &fp->fields[i]; - if (!field->enabled) { - continue; - } - - os_class_score_matched = field_score_db->match(field_score_db->data, field->value, field->value_len); - if (os_class_score_matched == NULL) { - continue; - } - - importance = osfp_fingerprint_get_field_importance(i); - - for (j = 0; j < OSFP_OS_CLASS_MAX; j++) { - entry_count = score_db->os_class_entry_count[j]; - tmp_score = os_class_score_matched->scores[j]; - tmp_score = tmp_score < (entry_count * FIELD_VALUE_DUP_RATE_MAX) ? tmp_score : (entry_count * FIELD_VALUE_DUP_RATE_MAX); - if (entry_count == 0 || tmp_score == 0) { - continue; - } - if (0 == flags || flags & OSFP_BIT_U32(j)) { - osfp_log_debug("%s %s: ((%d * %u / %u) * %u ) / %u\n", osfp_fingerprint_get_field_name(i), osfp_os_class_id_to_name(j), OSFP_PERCENTILE, importance, perfect_score, os_class_score_matched->scores[j], entry_count); - result_score->scores[j] += ((OSFP_PERCENTILE * importance / perfect_score) * tmp_score) / entry_count; - } - } - - if (i == OSFP_FIELD_TCP_OPTIONS) { - // if OSFP_FIELD_TCP_OPTIONS matched OSFP_FIELD_TCP_OPTIONS_ORDERED is not needed - i++; - } - } - - return OSFP_NOERR; -exit: - return ret; -} - -void osfp_score_db_debug_print(struct osfp_score_db *score_db) -{ - int i; - printf("score_db:\n"); - printf("entry_count: %u\n", score_db->entry_count); - printf("perfect_score: %u\n", score_db->perfect_score); - - for (i = 0; i < OSFP_OS_CLASS_MAX; i++) { - const char *name = osfp_os_class_id_to_name(i); - printf("os class %s ", name); - - printf("entry_count: %u\n", score_db->os_class_entry_count[i]); - printf("os class %s entry_count: %u\n", osfp_os_class_id_to_name(i), score_db->os_class_entry_count[i]); - } - - for (i = 0; i < OSFP_FIELD_MAX; i++) { - printf("field %s enabled: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].enabled); - printf("field %s type: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].type); - printf("field %s entry_count: %u\n", osfp_fingerprint_get_field_name(i), score_db->field_score_dbs[i].entry_count); - } -} - struct osfp_score_db *osfp_score_db_create(void) { int i; @@ -562,6 +509,12 @@ struct osfp_score_db *osfp_score_db_create(void) continue; } + score_db->field_weight[i] = osfp_fingerprint_get_field_importance(i); + // tcp options ordered and tcp options overlap + if (i != OSFP_FIELD_TCP_OPTIONS_ORDERED) { + score_db->total_weight += score_db->field_weight[i]; + } + db->type = osfp_fingerprint_get_field_type(i); switch (db->type) { case OSFP_FIELD_TYPE_UINT: @@ -577,13 +530,13 @@ struct osfp_score_db *osfp_score_db_create(void) db->match = osfp_score_db_hash_match; break; default: - osfp_log_debug("fingerprint field unsupported type: %u", db->type); + osfp_log_error("fingerprint field unsupported type: %u", db->type); goto exit; } db->data = db->create(); if (db->data == NULL) { - osfp_log_debug("field db create failed. field: %s", osfp_fingerprint_get_field_name(i)); + osfp_log_error("field db create failed. field: %s", osfp_fingerprint_get_field_name(i)); goto exit; } } @@ -680,7 +633,7 @@ int test_osfp_score_db(void) goto exit; } - if (db->os_class_entry_count[OSFP_OS_CLASS_ANDROID] != 1) { + if (db->os_entry_count[OSFP_OS_CLASS_ANDROID] != 1) { goto exit; } diff --git a/src/osfp_score_db.h b/src/osfp_score_db.h index cda21f0..acb8835 100644 --- a/src/osfp_score_db.h +++ b/src/osfp_score_db.h @@ -9,6 +9,10 @@ struct osfp_os_class_score { unsigned int scores[OSFP_OS_CLASS_MAX]; }; +struct osfp_field_value_count { + unsigned int counts[OSFP_OS_CLASS_MAX]; +}; + struct osfp_field_score_db { unsigned int enabled; unsigned int type; @@ -18,34 +22,38 @@ struct osfp_field_score_db { void *(*create)(void); void (*destroy)(void *); - int (*add)(void *data, struct osfp_os_class_score *, void *, unsigned int); - struct osfp_os_class_score *(*match)(void *, void *, unsigned int); + int (*add)(void *data, struct osfp_field_value_count *, void *, unsigned int); + struct osfp_field_value_count *(*match)(void *, void *, unsigned int); }; struct osfp_prefilter_hash_element { - struct osfp_fingerprint *fp; - char *fp_json; unsigned int os_class; - unsigned int repeated; + + struct osfp_fingerprint *fp; + char *fp_json; + UT_hash_handle hh; }; struct osfp_score_db { - unsigned int entry_count; - struct osfp_prefilter_hash_element *prefilter_head; + unsigned int entry_count; + unsigned int os_entry_count[OSFP_OS_CLASS_MAX]; + unsigned int perfect_score; - unsigned int os_class_entry_count[OSFP_OS_CLASS_MAX]; + unsigned int total_weight; + unsigned int field_weight[OSFP_FIELD_MAX]; + struct osfp_field_score_db field_score_dbs[OSFP_FIELD_MAX]; }; -char *osfp_score_db_read_file(char *fp_file); void osfp_score_db_debug_print(struct osfp_score_db *score_db); +const char *osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score); + int osfp_score_db_load(struct osfp_score_db *score_db, char *fp_file); -int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score); int osfp_score_db_score(struct osfp_score_db *score_db, unsigned int flags, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score); struct osfp_score_db *osfp_score_db_create(void); diff --git a/test/test.c b/test/test.c index 3cb0206..7a04228 100644 --- a/test/test.c +++ b/test/test.c @@ -132,7 +132,7 @@ void test_data_prepare() char *file_buffer; if (test_file_path == NULL) { - file_buffer = osfp_score_db_read_file(data_file_path); + file_buffer = osfp_read_file(data_file_path); if (file_buffer == NULL) { osfp_log_error("read file: '%s'\n", data_file_path); exit(1); @@ -188,7 +188,7 @@ void test_miss_rate() unsigned int fingerprint_count = 0; char *file_buffer; - file_buffer = osfp_score_db_read_file(test_file_path); + file_buffer = osfp_read_file(test_file_path); if (file_buffer == NULL) { osfp_log_error("read file: '%s'\n", test_file_path); exit(1);