1、调整SFH的函数名和源文件名,原有使用sfh的用户会受到影响;2、digest_gen由目录遍历,改为单文件并在屏幕输出结果,便于通过命令行调用。

This commit is contained in:
zhengchao
2017-07-08 19:23:17 +08:00
parent 5ba84a69f1
commit 6ffc3e3ded
7 changed files with 78 additions and 115 deletions

View File

@@ -1500,7 +1500,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
fuzzy_handle_t * tmp_fuzzy_handle=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct _Maat_table_info_t *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
if(p_table==NULL)
@@ -1508,7 +1508,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
_feather->scan_err_cnt++;
return NULL;
}
tmp_fuzzy_handle=fuzzy_create_handle(total_len);
tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL)
{
_feather->scan_err_cnt++;
@@ -1584,7 +1584,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
sp->acc_scan_len+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
do_query=REACH_QUERY_THRESH(sp->total_len, sp->acc_scan_len, sp->query_point,8);
if(do_query==0)
@@ -1592,7 +1592,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
goto fast_out;
}
pthread_mutex_lock(&(sp->fuzzy_mutex));
digest_len=fuzzy_status(sp->fuzzy_hash_handle, HASH_LENGTH);
digest_len=SFH_status(sp->fuzzy_hash_handle, HASH_LENGTH);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
if(digest_len==0)
{
@@ -1600,7 +1600,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
digest_buff=(char*)malloc(sizeof(char)*digest_len);
pthread_mutex_lock(&(sp->fuzzy_mutex));
fuzzy_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
if(GIE_handle!=NULL)
@@ -1661,7 +1661,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
DEC_SCANNER_REF(scanner, sp->thread_num);
}
}
fuzzy_destroy_handle(sp->fuzzy_hash_handle);
SFH_release(sp->fuzzy_hash_handle);
pthread_mutex_destroy(&(sp->fuzzy_mutex));
assert(sp->last_cache==NULL);
assert(sp->scan_buff==NULL);

View File

@@ -12,7 +12,7 @@
#include "rulescan.h"
#include "hiredis.h"
#include "mesa_fuzzy.h"
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
#include "aligment_int64.h"
#include <pthread.h>
@@ -314,7 +314,7 @@ struct _stream_para_t
void* rs_stream_para;
long acc_scan_len;
unsigned long long total_len;
fuzzy_handle_t *fuzzy_hash_handle;
sfh_instance_t *fuzzy_hash_handle;
pthread_mutex_t fuzzy_mutex;
unsigned char query_point[8];
};

View File

@@ -19,7 +19,7 @@ LIBMAAT = libmaatframe.a
LIBMAAT_SO = libmaatframe.so
OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_command.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o\
cJSON.o json2iris.o map_str2int.o interval_index.o gram_index_engine.o mesa_fuzzy.o rbtree.o
cJSON.o json2iris.o map_str2int.o interval_index.o gram_index_engine.o stream_fuzzy_hash.o rbtree.o
.c.o:
$(CC) -c $(CFLAGS) -I. $(H_DIR) $<

View File

@@ -1,6 +1,6 @@
#include<zt_hash.h>
#include<interval_index.h>
#include<mesa_fuzzy.h>
#include "zt_hash.h"
#include "interval_index.h"
#include "stream_fuzzy_hash.h"
#ifndef __SFH_INTERNAL_H_INCLUDE_
#define __SFH_INTERNAL_H_INCLUDE_
@@ -106,6 +106,5 @@ unsigned int segment_overlap(fuzzy_handle_inner_t * handle, unsigned int size, u
void sfh_tune_callback(IVI_seg_t * seg, void * user_para);
void sfh_output_callback(IVI_seg_t * seg, void * user_para);
void fuzzy_hash_length(IVI_seg_t * seg, void * user_para);
unsigned long long fuzzy_status(fuzzy_handle_t * handle, int type);
#endif

View File

@@ -72,7 +72,7 @@ static inline unsigned int sum_hash(unsigned char c, unsigned int h)
/**
* handle
*/
fuzzy_handle_t * fuzzy_create_handle(unsigned long long origin_len)
sfh_instance_t * SFH_instance(unsigned long long origin_len)
{
fuzzy_handle_inner_t * handle = NULL;
unsigned long long tmp_blksize = 0;
@@ -94,7 +94,7 @@ fuzzy_handle_t * fuzzy_create_handle(unsigned long long origin_len)
//handle->blocksize=tmp_blksize;
handle->blocksize = 3;
handle->do_tune=1;
return (fuzzy_handle_t *)handle;
return (sfh_instance_t *)handle;
}
@@ -110,7 +110,7 @@ void fuzzy_node_free(IVI_seg_t * seg, void * usr_para)
}
void fuzzy_destroy_handle(fuzzy_handle_t * handle)
void SFH_release(sfh_instance_t * handle)
{
IVI_destroy(((fuzzy_handle_inner_t *)handle)->ivi, fuzzy_node_free, (void *)handle);
((fuzzy_handle_inner_t *)handle)->fuzzy_node_memory -= sizeof(fuzzy_handle_inner_t);
@@ -118,7 +118,7 @@ void fuzzy_destroy_handle(fuzzy_handle_t * handle)
return;
}
unsigned int fuzzy_feed(fuzzy_handle_t * handle, const char * data, unsigned int size, unsigned long long offset)
unsigned int SFH_feed(sfh_instance_t * handle, const char * data, unsigned int size, unsigned long long offset)
{
fuzzy_handle_inner_t * _handle=(fuzzy_handle_inner_t *)handle;
if(data == NULL || size == 0)
@@ -150,7 +150,7 @@ unsigned int fuzzy_feed(fuzzy_handle_t * handle, const char * data, unsigned int
//printf("blocksize after:%llu\n", _handle->blocksize);
}
#if 0
fuzzy_digest(handle,result, sizeof(result));
SFH_digest(handle,result, sizeof(result));
printf("%llu %s\n",offset,result);
#endif
return length;
@@ -575,7 +575,7 @@ int sfh_merge_seg(fuzzy_handle_inner_t * _handle, sfh_seg_t * p, sfh_seg_t * n,u
/**
* hash_result值result输出abc[1:100]def[200:300]
*/
int fuzzy_digest(fuzzy_handle_t * handle, char * hash_buffer, unsigned int size)
int SFH_digest(sfh_instance_t * handle, char * hash_buffer, unsigned int size)
{
fuzzy_handle_inner_t* _handle=(fuzzy_handle_inner_t *)handle;
unsigned int estimate_len=_handle->s_state_cnt+IVI_seg_cnt(_handle->ivi)*24+1;
@@ -666,7 +666,7 @@ void sfh_output_callback(IVI_seg_t * seg, void * user_para)
/**
* fuzzy_hash的各种长度
*/
unsigned long long fuzzy_status(fuzzy_handle_t * handle, int type)
unsigned long long SFH_status(sfh_instance_t * handle, int type)
{
unsigned long long length;
fuzzy_handle_inner_t * _handle = (fuzzy_handle_inner_t *)(handle);