增加性能测试用例,处理无法获取redis time的异常。

This commit is contained in:
zhengchao
2019-07-28 19:13:04 +06:00
parent 718bbc59dd
commit 3473cf6dd4
8 changed files with 278 additions and 111 deletions

View File

@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.5)
set(MAAT_FRAME_MAJOR_VERSION 2)
set(MAAT_FRAME_MINOR_VERSION 7)
set(MAAT_FRAME_MINOR_VERSION 8)
set(MAAT_FRAME_PATCH_VERSION 0)
set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})

View File

@@ -143,9 +143,11 @@ long long redis_server_time(redisContext* ctx)
long long server_time=0;
redisReply* data_reply=NULL;
data_reply=_wrap_redisCommand(ctx,"TIME");
assert(data_reply->type==REDIS_REPLY_ARRAY);
server_time=atoll(data_reply->element[0]->str);
freeReplyObject(data_reply);
if(data_reply->type==REDIS_REPLY_ARRAY)
{
server_time=atoll(data_reply->element[0]->str);
freeReplyObject(data_reply);
}
return server_time;
}
enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
@@ -1434,7 +1436,10 @@ void check_maat_expiration(redisContext *ctx, void *logger)
long long server_time=0;
server_time=redis_server_time(ctx);
if(!server_time)
{
return;
}
data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time);
if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0)
{
@@ -1473,7 +1478,10 @@ void cleanup_update_status(redisContext *ctx, void *logger)
long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0;
server_time=redis_server_time(ctx);
if(!server_time)
{
return;
}
reply=_wrap_redisCommand(ctx,"MULTI");
freeReplyObject(reply);
redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
@@ -2103,6 +2111,10 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
{
return -1;
}
server_time=redis_server_time(write_ctx);
if(!server_time)
{
return -1;
}
s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num);
for(i=0;i<line_num;i++)
@@ -2392,7 +2404,10 @@ int Maat_cmd_commit(Maat_feather_t feather)
serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num);
p=p->next;
}
_feather->server_time=redis_server_time(write_ctx);
_feather->server_time=redis_server_time(write_ctx);
if(!_feather->server_time)
{
goto error_out;
}
if(_feather->AUTO_NUMBERING_ON==1)
{
@@ -2612,6 +2627,10 @@ static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAA
static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line)
{
redisContext* write_ctx=get_redis_ctx_for_write(_feather);
_feather->server_time=redis_server_time(write_ctx);
if(!_feather->server_time)
{
return -1;
}
struct serial_rule_t s_rule;
set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);

View File

@@ -33,7 +33,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_7_20190629=1;
int MAAT_FRAME_VERSION_2_8_20190728=1;
int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
@@ -3169,26 +3169,39 @@ void vector_print(igraph_vector_t *v) {
}
printf("\n");
}
static size_t effective_vertices_count(igraph_vector_t *vids)
{
size_t i=0;
int tmp_vid=0;
for(i=0; i<(size_t)igraph_vector_size(vids); i++)
{
tmp_vid=(int) VECTOR(*vids)[i];
if(tmp_vid<0)
{
break;
}
}
return i;
}
void walk_group_hash(const uchar * key, uint size, void * data, void * user)
{
struct Maat_group_inner* group_rule=(struct Maat_group_inner*)data;
struct Maat_group_inner* parent_group=NULL;
struct Maat_scanner* scanner=(struct Maat_scanner*)user;
int tmp_vid=0;
igraph_vector_t vids;
igraph_vector_init(&vids, 0);
igraph_vector_t *vids=&(scanner->dfs_vids);
igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT,
0, &vids, NULL, NULL, NULL, NULL, NULL, NULL);
0, vids, NULL, NULL, NULL, NULL, NULL, NULL);
long int i=0;
long long* temp_group_ids=ALLOC(long long, igraph_vector_size(&vids));
size_t top_group_cnt=0;
for(i=0; i<igraph_vector_size(&vids); i++)
size_t i=0, top_group_cnt=0;
size_t parent_group_cnt=effective_vertices_count(vids);
long long* temp_group_ids=ALLOC(long long, parent_group_cnt);
for(i=0; i<(size_t)igraph_vector_size(vids); i++)
{
tmp_vid=(int) VECTOR(vids)[i];
tmp_vid=(int) VECTOR(*vids)[i];
if(tmp_vid<0)
{
break;
@@ -3196,10 +3209,11 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user)
parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid);
if(parent_group->has_compile_neighbors)//including itself
{
temp_group_ids[top_group_cnt]=parent_group->group_id;
temp_group_ids[top_group_cnt]=parent_group->group_id;
top_group_cnt++;
}
}
pthread_mutex_lock(&(group_rule->mutex));
free(group_rule->top_groups);
group_rule->top_group_cnt=top_group_cnt;
@@ -3211,7 +3225,6 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user)
scanner->most_popular_sub_group=group_rule->group_id;
}
pthread_mutex_unlock(&(group_rule->mutex));
igraph_vector_destroy(&vids);
free(temp_group_ids);
temp_group_ids=NULL;
return;
@@ -3219,7 +3232,10 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user)
void find_group_paths(struct Maat_scanner* scanner)
{
scanner->group_graph_vcount=igraph_vcount(&scanner->group_graph);
igraph_vector_init(&(scanner->dfs_vids), scanner->group_graph_vcount);
MESA_htable_iterate(scanner->group_hash, walk_group_hash, scanner);
igraph_vector_destroy(&scanner->dfs_vids);
return;
}
void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, int scan_thread_num, void* logger)

View File

@@ -249,6 +249,9 @@ struct Maat_scanner
MESA_htable_handle vertex_id2group;
igraph_t group_graph;
igraph_integer_t group_graph_vcount;
igraph_vector_t dfs_vids;
int grp_vertex_id_generator;
int most_popular_sub_group;
unsigned long long max_presented_top_group_cnt;