gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge tag 'v1.8'

Browse Source 
修复可导致误命中的严重错误

# Conflicts:
#	inc/Maat_rule.h
#	src/entry/Maat_rule.cpp
This commit is contained in:
zhengchao
2016-06-03 15:21:31 +08:00
parent 0f1bc896cf a591c06b2f
commit 89541b66a1
31 changed files with 4496 additions and 1457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
inc/Maat_rule.h
View File

@@ -13,11 +13,9 @@
*/
#ifndef H_MAAT_RULE_H_INCLUDE
#define H_MAAT_RULE_H_INCLUDE
#ifndef __cplusplus
#error("This file should be compiled with C++ compiler")
#endif
#include "stream.h"
enum MAAT_CHARSET
{
@@ -26,7 +24,13 @@ enum MAAT_CHARSET
CHARSET_BIG5,
CHARSET_UNICODE,
CHARSET_UTF8, // 4
CHARSET_BIN //5
CHARSET_BIN, //5
CHARSET_UNICODE_ASCII_ESC, // Unicode Escape format, prefix backslash-u hex, e.g. "\u627;"
CHARSET_UNICODE_ASCII_ALIGNED,//Unicode Escape format, prefix backslash-u with 4 bytes aligned, e.g. "\u0627"
CHARSET_UNICODE_NCR_DEC, //SGML Numeric character reference,decimal base, e.g. "ا"
CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا"
CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA
CHARSET_URL_ENCODE_UTF8 //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD
};
enum MAAT_ACTION
{
@@ -113,17 +117,37 @@ struct Maat_hit_detail_t
struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM];
};
//--------------------HITTING DETAIL DESCRIPTION END
//Abondon interface ,left for compatible.
Maat_feather_t Maat_summon_feather(int max_thread_num,
const char* table_info_path,
const char* ful_cfg_dir,
const char* inc_cfg_dir,
void*logger);
void*logger);//MESA_handle_logger
//Abondon interface ,left for compatible.
Maat_feather_t Maat_summon_feather_json(int max_thread_num,
const char* table_info_path,
const char* json_rule,
void* logger);
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger);
int Maat_initiate_feather(Maat_feather_t feather);
enum MAAT_INIT_OPT
{
MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger,SIZE=sizeof(int). DEFAULT:1,000 milliseconds.
MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger,SIZE=sizeof(int). DEFAULT:60,000 milliseconds.
MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_STAT_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF.
MAAT_OPT_PERF_ON, //VALUE is indifferent,SIZE is indifferent.MAAT_OPT_STAT_FILE_PATH must be set.Default: stat OFF.
MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
MAAT_OPT_SCAN_DETAIL //VALUE is interger,SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping;
// 2 return hit pos and regex grouping pos;DEFAULT:2
};
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
void Maat_burn_feather(Maat_feather_t feather);
//return table_id(>=0) if success,otherwise return -1;
@@ -135,7 +159,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
Maat_finish_callback_t *finish,//u_para
void* u_para);
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,MUST end with '\0',SIZE= strlen(string+'\0')+1.DEFAULT: no default.
};
//return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size);
//Return hit rule number, return -1 when error occurs,return -2 when hit current region
//mid MUST set NULL before fist call
@@ -184,10 +213,5 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para);
void Maat_clean_status(scan_status_t* mid);
#define MAAT_OPT_SCANDIR_INTERVAL_MS 1 //opt value is interger,size=sizeof(int),1,000 milliseconds as default
#define MAAT_OPT_EFFECT_INVERVAL_MS 2 //opt value is interger,size=sizeof(int),60,000 milliseconds as default
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,int type,void* value,int size);
#endif // H_MAAT_RULE_H_INCLUDE

0
src/inc_internal/mesa_fuzzy.h → inc/mesa_fuzzy.h
View File

644
src/entry/Maat_api.cpp
View File

File diff suppressed because it is too large Load Diff

429
src/entry/Maat_rule.cpp
View File

@@ -11,11 +11,13 @@
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
#include <MESA/MESA_handle_logger.h>
#include <MESA/field_stat2.h>
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "json2iris.h"
#include "dynamic_array.h"
#include "aligment_int64.h"
#include "config_monitor.h"
#include "map_str2int.h"
@@ -24,11 +26,11 @@
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_3_20160131=1;
int MAAT_FRAME_VERSION_1_8_20160603=1;
const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"CHARSET_NONE","GBK","BIG5","UNICODE","UTF-8"};
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
int converHextoint(char srctmp)
{
if(isdigit(srctmp))
@@ -78,6 +80,7 @@ iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enu
cd=scanner->iconv_handle[to][from];
return cd;
}
int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{
size_t ret;
@@ -103,7 +106,8 @@ int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MA
if(ret!=(size_t)(-1))
{
if(to==CHARSET_UNICODE)//jump unicode 2 bytes head 0xFF 0xFE
if(to==CHARSET_UNICODE&&
(*(unsigned short*)pOutBuff==0xFFFE||*(unsigned short*)pOutBuff==0XFEFF))//jump unicode 2 bytes BOM, 0xFF 0xFE
{
copy_len=iOutBuffLen-iLeftLen-2;
copy_buf=pOutBuff+2;
@@ -132,7 +136,137 @@ int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MA
}
}
int URLEncode(const char* str, const int strSize, char* result, const int resultSize)
{
int i;
int j = 0;//for result index
char ch;
if ((str==NULL) || (result==NULL) || (strSize<=0) || (resultSize<=0))
{
return -1;
}
for ( i=0; (i<strSize)&&(j<resultSize); ++i)
{
ch = str[i];
if (((ch>='A') && (ch<'Z')) ||
((ch>='a') && (ch<'z')) ||
((ch>='0') && (ch<'9')))
{
result[j++] = ch;
}
else if (ch == ' ')
{
result[j++] = '+';
}
else if (ch == '.' || ch == '-' || ch == '_' || ch == '*')
{
result[j++] = ch;
}
else
{
if (j+3 < resultSize)
{
sprintf(result+j, "%%%02X", (unsigned char)ch);
j += 3;
}
else
{
return -1;
}
}
}
result[j] = '\0';
return j;
}
int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, const int dstsize)
{
int i=0,j=0;
assert(srclen%2==0);//unicode must be 2 bytes aligned.
while(i<srclen&&j<dstsize)
{
if(*(unsigned short*)(src+i)<0x7f)
{
dst[j]=*(unsigned short*)(src+i);
j++;
}
else
{
j+=snprintf(dst+j,dstsize-j,fmt,*(unsigned short*)(src+i));
}
i+=2;
}
return j;
}
int universal_charset_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{
int ret=0;
char* tmp_buff=NULL;
int tmp_buff_size=0;
MAAT_CHARSET tmp_dst_code=CHARSET_NONE;
const char* fmt=NULL;
switch(to)
{
case CHARSET_GBK:
case CHARSET_BIG5:
case CHARSET_UNICODE:
case CHARSET_UTF8:
ret=iconv_convert(scanner,from,to,src,srclen,dst,dstlen);
return ret;
break;
case CHARSET_UNICODE_ASCII_ESC:
tmp_dst_code=CHARSET_UNICODE;
fmt="\\u%x;";
break;
case CHARSET_UNICODE_ASCII_ALIGNED:
tmp_dst_code=CHARSET_UNICODE;
fmt="\\u%04x";
break;
case CHARSET_UNICODE_NCR_DEC:
tmp_dst_code=CHARSET_UNICODE;
fmt="&#%u;";
break;
case CHARSET_UNICODE_NCR_HEX:
tmp_dst_code=CHARSET_UNICODE;
fmt="&#x%x;";
break;
case CHARSET_URL_ENCODE_GB2312:
tmp_dst_code=CHARSET_GBK;
fmt=NULL;
break;
case CHARSET_URL_ENCODE_UTF8:
tmp_dst_code=CHARSET_UTF8;
fmt=NULL;
break;
default:
return -1;
break;
}
tmp_buff_size=*dstlen;
tmp_buff=(char*)malloc(tmp_buff_size);
ret=iconv_convert(scanner,from,tmp_dst_code,src,srclen,tmp_buff,&tmp_buff_size);
if(ret<0)
{
goto error_out;
}
if(fmt!=NULL)
{
ret=uni2ascii(fmt, tmp_buff, tmp_buff_size, dst,*dstlen);
}
else
{
ret=URLEncode(tmp_buff,tmp_buff_size,dst,*dstlen);
}
*dstlen=ret;
error_out:
free(tmp_buff);
tmp_buff=NULL;
return ret;
}
char* strlwr(char* string)
{
int i=0;
@@ -218,17 +352,7 @@ int cnt_maskbits(struct in6_addr mask)
}
return bits_cnt;
}
int aligment_int_array_sum(int * array,int size)
{
int sum=0,i=0;
int offset=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(int))*i;
sum+=array[offset];
}
return sum;
}
int lqueue_destroy_cb(void *data, long data_len, void *arg)
{
assert(0);
@@ -253,12 +377,34 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void* logger)
_Maat_table_info_t* create_table_info(int max_thread_num)
{
struct _Maat_table_info_t*p=NULL;
p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1);
p->scan_cnt=aligment_int64_array_alloc(max_thread_num);
p->scan_cpu_time=aligment_int64_array_alloc(max_thread_num);
p->input_bytes=aligment_int64_array_alloc(max_thread_num);
p->stream_num=aligment_int64_array_alloc(max_thread_num);
p->hit_cnt=aligment_int64_array_alloc(max_thread_num);
p->cross_cache_size=0;
return p;
}
void destroy_table_info(struct _Maat_table_info_t*p)
{
aligment_int64_array_free(p->scan_cnt);
aligment_int64_array_free(p->scan_cpu_time);
aligment_int64_array_free(p->input_bytes);
aligment_int64_array_free(p->stream_num);
aligment_int64_array_free(p->hit_cnt);
free(p);
return;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{
FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE];
int i=0,j=0,ret[4]={0},table_cnt=0;
char table_type[16],src_charset[16],dst_charset[64],merge[4];
char table_type[16],src_charset[256],dst_charset[256],merge[4];
MESA_htable_handle string2int_map=map_create();
char *token=NULL,*sub_token=NULL,*saveptr;
struct _Maat_table_info_t*p=NULL;
@@ -269,12 +415,28 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTVAL);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"bin", CHARSET_NONE);
for(i=0;i<MAX_CHARSET_NUM;i++)
{
if(strlen(CHARSET_STRING[i])>0)
{
map_register(string2int_map,CHARSET_STRING[i], i);
}
else
{
break;
}
}
/*
map_register(string2int_map,"gbk", CHARSET_GBK);
map_register(string2int_map,"big5", CHARSET_BIG5);
map_register(string2int_map,"unicode", CHARSET_UNICODE);
map_register(string2int_map,"utf8", CHARSET_UTF8);
map_register(string2int_map,"unicode_hex", CHARSET_UNICODE_ASCII_ESC);
map_register(string2int_map,"unicode_hex", CHARSET_UNICODE_ASCII_ESC);
*/
map_register(string2int_map,"yes", 1);
map_register(string2int_map,"no", 0);
@@ -285,6 +447,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s error.\n",table_info_path);
}
i=0;
while(NULL!=fgets(line,sizeof(line),fp))
{
i++;
@@ -293,8 +456,8 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
{
continue;
}
p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1);
p->cross_cache_size=0;
p=create_table_info(max_thread_num);
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id)
,p->table_name
,table_type
@@ -309,9 +472,9 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
{
if(ret[j]<0)
{
fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error.\n",table_info_path,i);
"Maat read table info %s line %d error:unknown column.\n",table_info_path,i);
goto error_jump;
}
}
@@ -332,9 +495,9 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
}
else
{
fprintf(stderr,"Maat read table info %s line %d error.\n",table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:unknown dest charset %s.\n",table_info_path,i,sub_token);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error.\n",table_info_path,i);
"Maat read table info %s line %d error: unknown dest charset %s.\n",table_info_path,i,sub_token);
goto error_jump;
}
@@ -365,7 +528,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
table_cnt++;
continue;
error_jump:
free(p);
destroy_table_info(p);
p=NULL;
}
fclose(fp);
@@ -601,6 +764,7 @@ struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para
{
struct op_expr_t* op_expr=NULL;
op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1);
op_expr->no_effect_convert_cnt=0;
op_expr->p_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),1);
op_expr->p_expr->expr_id=expr_id;
op_expr->p_expr->operation=operation;
@@ -673,7 +837,7 @@ void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL;
return;
}
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb)
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb,int rs_scan_type)
{
int i=0;
@@ -682,7 +846,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
hargs.thread_safe=0;
hargs.hash_slot_size = 1024*1024;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL;
@@ -706,14 +870,17 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->region_hash,0);
scanner->district_map=map_create();
scanner->version=version;
scanner->cfg_num=0;
scanner->max_thread_num=scan_thread_num;
//optimized for CPU cache_alignment 64
scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num);
scanner->ref_cnt=aligment_int64_array_alloc(scan_thread_num);
scanner->region_update_q=MESA_lqueue_create(0,0);
scanner->region=rulescan_initialize(scan_thread_num);
rulescan_set_param(scanner->region,rs_scan_type);
scanner->tomb_ref=tomb;
scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num);
@@ -737,7 +904,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
MESA_htable_destroy(scanner->compile_hash,NULL);
MESA_htable_destroy(scanner->group_hash, NULL);
MESA_htable_destroy(scanner->region_hash, NULL);
map_destroy(scanner->district_map);
destroy_bool_matcher((void*)scanner->expr_compiler);
q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
for(i=0;i<q_cnt;i++)
@@ -917,11 +1084,12 @@ void batch_update(struct _Maat_scanner_t *scanner,void*logger)
scanner->last_update_time=time(NULL);
return;
}
struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
struct _Maat_group_rule_t* add_region_to_group(struct _Maat_group_rule_t* group,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
{
struct _Maat_region_rule_t* region_rule=(struct _Maat_region_rule_t*)malloc(sizeof(struct _Maat_region_rule_t));
region_rule->region_id=region_id;
region_rule->expr_id=expr_id;
region_rule->district_id=district_id;
region_rule->region_type=region_type;
pthread_mutex_lock(&(group->mutex));
dynamic_array_write(group->region_rules,group->region_boundary,region_rule);
@@ -1075,12 +1243,24 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
}
return 1;
}
int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
{
int map_ret=0,district_id=-1;
map_ret=map_str2int(scanner->district_map, district_str,&district_id);
if(map_ret<0)
{
district_id= scanner->district_num;
map_register(scanner->district_map,district_str, district_id);
scanner->district_num++;
}
return district_id;
}
int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger)
{
unsigned int i=0,j=0;
char* p=NULL,*saveptr=NULL,*region_string=NULL;
int region_str_len=0,ret=0,k=0;
int expr_id=0;
int expr_id=0,district_id=-1;
scan_rule_t*p_rule=NULL;
struct _Maat_group_rule_t* group_rule=NULL;
@@ -1095,7 +1275,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
int sub_expr_cnt=0;
struct op_expr_t *op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{
assert(strlen(db_rule->district)>0);
district_id=get_district_id(scanner, db_rule->district);
}
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL)
{
@@ -1167,7 +1352,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break;
case EXPR_TYPE_REGEX://it's easy,no need to charset convert
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_rule->region_id,expr_id,TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule,db_rule->region_id,district_id,expr_id,TABLE_TYPE_EXPR);
if(u_para==NULL)
{
return -1;
@@ -1233,7 +1418,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
break;
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL)//duplicate
{
return -1;
@@ -1248,12 +1433,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
{
continue;
}
region_str_len=strlen(sub_key_array[k])*2+1;
region_str_len=strlen(sub_key_array[k])*8+1; // 1 byte map to 8 bytes maximum, e.g. "&#x0627;" or "\u63221;"
region_string=(char*)calloc(sizeof(char),region_str_len);
if(table->src_charset!=dst_charset)//need convert
{
ret=iconv_convert(scanner,table->src_charset, dst_charset,
ret=universal_charset_convert(scanner,table->src_charset, dst_charset,
sub_key_array[k],strlen(sub_key_array[k]),
region_string, &region_str_len);
if(ret<0)
@@ -1266,13 +1451,10 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
free(region_string);
continue;
}
//if convert take no effect
if(region_str_len==(int)strlen(sub_key_array[k])&&
0==memcmp(sub_key_array[k],region_string,region_str_len)&&
TRUE==table->src_charset_in_dst)
0==memcmp(sub_key_array[k],region_string,region_str_len))
{
free(region_string);
continue;
op_expr->no_effect_convert_cnt++;;
}
}
else
@@ -1291,14 +1473,23 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
free(region_string);
region_string=NULL;
}
MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*));
//if each sub string's convert take no effect and src charset is one of the dst.
if(TRUE==table->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)
{
destroy_op_expr(op_expr);
op_expr=NULL;
}
else
{
MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*));
}
}
}
else
{
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule, db_rule->region_id,expr_id, TABLE_TYPE_EXPR);
u_para=add_region_to_group(group_rule, db_rule->region_id,district_id,expr_id, table->table_type);
if(u_para==NULL)
{
return -1;
@@ -1334,7 +1525,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL)
@@ -1344,7 +1535,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_ip_rule->region_id,expr_id,TABLE_TYPE_IP);
u_para=add_region_to_group(group_rule,db_ip_rule->region_id,district_id,expr_id,TABLE_TYPE_IP);
if(u_para==NULL)
{
return -1;
@@ -1365,8 +1556,8 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL)
{
@@ -1374,7 +1565,7 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,intval_rule->region_id,expr_id,TABLE_TYPE_INTVAL);
u_para=add_region_to_group(group_rule,intval_rule->region_id,district_id,expr_id,TABLE_TYPE_INTVAL);
if(u_para==NULL)
{
return -1;
@@ -1394,7 +1585,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db
struct _Maat_group_rule_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL;
struct _Maat_group_rule_t* u_para=NULL;
int expr_id=0;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_rule_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL)
@@ -1403,7 +1594,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db
HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,TABLE_TYPE_DIGEST);
u_para=add_region_to_group(group_rule,db_digest_rule->region_id,expr_id,district_id,TABLE_TYPE_DIGEST);
if(u_para==NULL)
{
return -1;
@@ -1607,7 +1798,8 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
if(db_group_rule.is_valid==FALSE)
{
del_group_rule(table, &db_group_rule,scanner,logger);
table->cfg_num--;
//leave no trace for virtual group rule
// table->cfg_num--;
}
else
{
@@ -1623,7 +1815,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
else
{
//no need to free db_group_rule,it was saved in scanner->compile_hash
table->cfg_num++;
// table->cfg_num++;
}
}
@@ -1642,20 +1834,46 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
{
struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t));
int ret=0,db_hexbin=0;
ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=7)
switch(table->table_type)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
case TABLE_TYPE_EXPR:
ret=sscanf(table_line,"%d\t%d\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=7)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
}
break;
case TABLE_TYPE_EXPR_PLUS:
ret=sscanf(table_line,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
,&(maat_str_rule->group_id)
,maat_str_rule->district
,maat_str_rule->keywords
,(int*)&(maat_str_rule->expr_type)
,(int*)&(maat_str_rule->match_method)
,&db_hexbin
,&(maat_str_rule->is_valid));
if(ret!=8)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of expr_plus table %s:%s",table->table_name,table_line);
free(maat_str_rule);
maat_str_rule=NULL;
return;
}
break;
default:
assert(0);
break;
}
switch(db_hexbin)
{
@@ -1830,6 +2048,14 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
if(ret>0)
{
table->cfg_num--;
if(ip_rule->addr_type==4)
{
table->ipv4_rule_cnt--;
}
else
{
table->ipv6_rule_cnt--;
}
}
}
else
@@ -1845,6 +2071,14 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
else
{
table->cfg_num++;
if(ip_rule->addr_type==4)
{
table->ipv4_rule_cnt++;
}
else
{
table->ipv6_rule_cnt++;
}
}
}
error_out:
@@ -2061,7 +2295,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
long data_size=0;
const long q_cnt=MESA_lqueue_get_count(garbage_q);
int i=0,bury_cnt=0;
int ref_cnt=0;
long long ref_cnt=0;
int have_timeout=0;
time_t now=time(NULL);
for(i=0;i<q_cnt;i++)
@@ -2084,7 +2318,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
destroy_group_rule(bag->group_rule);
break;
case GARBAGE_SCANNER:
ref_cnt=aligment_int_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
ref_cnt=aligment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
if(ref_cnt==0)
{
@@ -2094,7 +2328,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
else
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d force destroyed,ref_cnt %d.",
"scanner %p version %d force destroyed,ref_cnt %lld.",
bag->scanner,bag->scanner->version,ref_cnt);
}
@@ -2124,15 +2358,6 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li
struct _plugin_table_info* p_table_cb=table->cb_info;
char *p=NULL;
pthread_mutex_lock(&(p_table_cb->plugin_mutex));
if(p_table_cb->is_called_started==0)
{
for(i=0;i<p_table_cb->cb_plug_cnt;i++)
{
p_table_cb->cb_plug[i].start(p_table_cb->update_type,p_table_cb->cb_plug[i].u_para);
}
p_table_cb->is_called_started=1;
}
if(p_table_cb->cb_plug_cnt>0)
{
for(i=0;i<p_table_cb->cb_plug_cnt;i++)
@@ -2197,10 +2422,10 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
struct _Maat_table_info_t* p_table=NULL;
struct _plugin_table_info* p_table_cb=NULL;
int i=0;
int i=0,j=0;
if(update_type==CM_UPDATE_TYPE_FULL)
{
feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q);
feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q,feather->rule_scan_type);
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Full config version %u -> %u update start",
feather->maat_version,new_version);
@@ -2213,7 +2438,7 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
feather->maat_version,new_version);
feather->maat_version=new_version;
}
for(i=0;i<MAX_TABLE_NUM;i++)
for(i=0;i<feather->table_cnt;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN)
@@ -2221,8 +2446,13 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
continue;
}
p_table_cb=p_table->cb_info;
p_table_cb->update_type=update_type;
assert(p_table_cb->is_called_started==0);
for(j=0;j<p_table_cb->cb_plug_cnt;j++)
{
if(p_table_cb->cb_plug[j].start!=NULL)
{
p_table_cb->cb_plug[j].start(update_type,p_table_cb->cb_plug[j].u_para);
}
}
}
return;
}
@@ -2249,17 +2479,13 @@ void maat_finish_cb(void* u_para)
continue;
}
p_table_cb=p_table->cb_info;
if(p_table_cb->is_called_started==0)
{
continue;
}
p_table_cb->is_called_started=0;
pthread_mutex_lock(&(p_table_cb->plugin_mutex));
for(j=0;j<p_table_cb->cb_plug_cnt;j++)
{
p_table_cb->cb_plug[j].finish(p_table_cb->cb_plug[j].u_para);
if(p_table_cb->cb_plug[j].finish!=NULL)
{
p_table_cb->cb_plug[j].finish(p_table_cb->cb_plug[j].u_para);
}
}
pthread_mutex_unlock(&(p_table_cb->plugin_mutex));
}
if(feather->update_tmp_scanner!=NULL)
{
@@ -2289,10 +2515,16 @@ void maat_finish_cb(void* u_para)
else
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Postpone config version %u (%d entries) load to rulescan.",
"Postpone config version %u %d entries load to rulescan.",
feather->scanner->version,feather->scanner->cfg_num);
}
}
else
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Version %d have no valid scan rules, plugin callback complete.",
feather->maat_version);
}
return;
}
void maat_update_cb(const char* table_name,const char* line,void *u_para)
@@ -2318,6 +2550,7 @@ void maat_update_cb(const char* table_name,const char* line,void *u_para)
switch(feather->p_table_info[table_id]->table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON);
break;
case TABLE_TYPE_IP:
@@ -2349,9 +2582,11 @@ void *thread_rule_monitor(void *arg)
const char* inc_cfg_dir=(const char*)feather->inc_dir;
struct _Maat_scanner_t* old_scanner=NULL;
long expr_wait_q_cnt=0;
int scan_dir_cnt=0;
while(feather->still_working)
{
usleep(feather->scan_interval_ms*1000);
scan_dir_cnt++;
config_monitor_traverse(feather->maat_version,
inc_cfg_dir,
maat_start_cb,
@@ -2389,12 +2624,17 @@ void *thread_rule_monitor(void *arg)
}
}
garbage_bury(feather->garbage_q,feather->logger);
if(feather->stat_on==1&&scan_dir_cnt%2==0)//output every 2 seconds
{
maat_stat_output(feather);
}
}
MESA_htable_destroy(feather->map_tablename2id,free);
destroy_maat_scanner(feather->scanner);
garbage_bury(feather->garbage_q,feather->logger);
MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL);
FS_stop(&(feather->stat_handle));
int i=0,j=0;
struct dynamic_array_t* d_array=NULL;
@@ -2414,9 +2654,14 @@ void *thread_rule_monitor(void *arg)
free(lines);
}
}
free(feather->p_table_info[i]);
destroy_table_info(feather->p_table_info[i]);
feather->p_table_info[i]=NULL;
}
aligment_int64_array_free(feather->thread_call_cnt);
aligment_int64_array_free(feather->inner_mid_cnt);
aligment_int64_array_free(feather->outer_mid_cnt);
aligment_int64_array_free(feather->hit_cnt);
free(feather);
return NULL;
}

80
src/entry/Maat_rule_internal.h
View File

@@ -5,13 +5,14 @@
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
#include <MESA/field_stat2.h>
#include "dynamic_array.h"
#include "UniversalBoolMatch.h"
#include "rulescan.h"
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
#include "aligment_int64.h"
#include <pthread.h>
#include <iconv.h>
@@ -30,15 +31,15 @@ typedef int atomic_t;
#else
#include <alsa/iatomic.h>
#endif
#define CPU_CACHE_ALIGMENT 64
#define TRUE 1
#define FALSE 0
#define MAX_TABLE_NUM 256
#define MAX_CHARSET_NUM 6
#define MAX_CHARSET_NUM 16
#define MAX_TABLE_NAME_LEN 256
#define MAX_TABLE_LINE_SIZE (1024*4)
#define MAX_EXPR_KEYLEN 1024
#define MAX_DISTRICT_LEN 64
#define MAX_PLUGING_NUM 32
#define MAX_SCANNER_HIT_NUM 64
@@ -47,6 +48,7 @@ typedef int atomic_t;
#define MAX_FAILED_NUM 128
#define MAX_MAAT_STAT_NUM 64
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
@@ -74,6 +76,7 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_PLUGIN,
TABLE_TYPE_INTVAL,
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_GROUP
};
@@ -91,11 +94,13 @@ enum MAAT_MATCH_METHOD
MATCH_METHOD_LEFT,
MATCH_METHOD_FULL
};
struct db_str_rule_t
{
int region_id;
int group_id;
char keywords[MAX_EXPR_KEYLEN];
char district[MAX_DISTRICT_LEN];
enum MAAT_EXPR_TYPE expr_type;
enum MAAT_MATCH_METHOD match_method;
int is_hexbin;
@@ -162,12 +167,14 @@ struct op_expr_t
{
boolean_expr_t* p_expr;
scan_rule_t* p_rules[MAAT_MAX_EXPR_ITEM_NUM];
int no_effect_convert_cnt;
};
struct _Maat_region_rule_t
{
int region_id;
int expr_id;
int district_id;
enum MAAT_TABLE_TYPE region_type;
};
struct _Maat_group_rule_t
@@ -223,20 +230,42 @@ struct _Maat_table_info_t
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int expr_rule_cnt; //expr_type=0,1,3
int regex_rule_cnt; //expr_type=2
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
struct _plugin_table_info *cb_info;
//for stat>>>>>>>>
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct _scan_status_t
struct _INNER_scan_status_t
{
struct _Maat_feather_t* feather;
int thread_num;
int cur_hit_cnt;
int hit_group_cnt;
int hit_group_size;
unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned int *hitted_group_id;
};
struct _OUTER_scan_status_t
{
struct _Maat_feather_t* feather;
unsigned short thread_num;
unsigned short is_set_district;
int district_id;
struct _INNER_scan_status_t* inner;
};
enum maat_garbage_type
{
GARBAGE_SCANNER=0,
@@ -273,13 +302,15 @@ struct _Maat_scanner_t
{
int version;
time_t last_update_time;
int *ref_cnt; //optimized for cache_alignment 64
long long *ref_cnt; //optimized for cache_alignment 64
rule_scanner_t region;
pthread_rwlock_t digest_rwlock[MAX_TABLE_NUM];
GIE_handle_t* digest_handle[MAX_TABLE_NUM];
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
MESA_htable_handle district_map;
unsigned int district_num;
unsigned int cfg_num;
unsigned int exprid_generator;
MESA_lqueue_head region_update_q;
@@ -296,17 +327,32 @@ struct _Maat_feather_t
struct _Maat_scanner_t *update_tmp_scanner;
MESA_lqueue_head garbage_q;
int table_cnt;
int GROUP_MODE_ON;
int still_working;
int scan_interval_ms;
int effect_interval_ms;
int stat_on;
int perf_on;
struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM];
MESA_htable_handle map_tablename2id;
void* logger;
int maat_version;
int scan_thread_num;
int rule_scan_type;
char inc_dir[MAX_TABLE_NAME_LEN];
char full_dir[MAX_TABLE_NAME_LEN];
int GROUP_MODE_ON;
int still_working;
int scan_interval_ms;
int effect_interval_ms;
char stat_file[MAX_TABLE_NAME_LEN];
//for stat>>>>
screen_stat_handle_t stat_handle;
int total_stat_id;
int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_MAAT_STAT_NUM];
mcore_long_t outer_mid_cnt;
mcore_long_t inner_mid_cnt;
mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
long long total_scan_bytes;
long long total_scan_cnt;
};
struct _maat_garbage_t
{
@@ -325,7 +371,7 @@ struct _maat_garbage_t
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void*logger);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
void maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para);
@@ -340,5 +386,11 @@ inline void ipv6_ntoh(unsigned int *v6_addr)
}
return;
}
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather);
#endif

253
src/entry/Maat_stat.cpp Normal file
View File

@@ -0,0 +1,253 @@
#include "Maat_rule_internal.h"
#include "aligment_int64.h"
#include <time.h>
#include <MESA/field_stat.h>
enum MAAT_FS_STATUS{
STATUS_VERSION=0,
STATUS_THRED_NUM,
STATUS_TABLE_NUM,
STATUS_OUTER_MID_NUM,
STATUS_INNER_MID_NUM,
STATUS_GARBAGE_QSIZE,
STATUS_TOTAL_SCAN_LEN,
STATUS_TOTAL_SCAN_CNT,
};
enum MAAT_FS_COLUMN
{
COLUMN_TABLE_RULE_NUM=0,
COLUMN_TABLE_REGEX_NUM,
COLUMN_TABLE_STREAM_NUM,
COLUMN_TABLE_SCAN_CNT,
COLUMN_TABLE_SCAN_BYTES,
COLUMN_TABLE_CPU_TIME,//microseconds
COLUMN_TABLE_HIT_CNT,
};
void maat_stat_init(struct _Maat_feather_t* feather)
{
int value=0;
int i=0;
struct _Maat_table_info_t* p_table=NULL;
feather->stat_handle=FS_create_handle();
FS_set_para(feather->stat_handle, OUTPUT_DEVICE, feather->stat_file, strlen(feather->stat_file)+1);
value=1;
FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value));
value=0;
FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value));
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps");
if(feather->perf_on==1)
{
feather->fs_column_id[COLUMN_TABLE_CPU_TIME]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"cpu_us");
value=feather->fs_column_id[COLUMN_TABLE_CPU_TIME];
FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
1000000, //microsecond to second
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"PROC_Bps");
}
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Tps");
if(feather->perf_on==1)
{
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
1000000, //microsecond to second
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"PROC_Tps");
}
feather->fs_column_id[COLUMN_TABLE_HIT_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"hit_cnt");
value=feather->fs_column_id[COLUMN_TABLE_HIT_CNT];
FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
1,
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"hit_rate");
feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum");
for(i=0;i<MAX_TABLE_NUM;i++)
{
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
p_table->stat_line_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,p_table->table_name);
}
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{
aligment_int64_array_add(p_table->scan_cnt,thread_num,1);
aligment_int64_array_add(p_table->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL)
{
aligment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
}
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
time(&now);
active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=aligment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=aligment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_stream_num=0;
table_scan_cnt=0;
table_input_bytes=0;
table_scan_cpu_time=0;
table_hit_cnt=0;
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
table_hit_cnt);
//total hit count stat in region_compile
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
total_cfg_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
total_regex_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
total_stream_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
total_scan_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
total_input_bytes);
if(feather->perf_on==1)
{
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
total_cpu_time);
}
total_hit_cnt=aligment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
total_hit_cnt);
feather->total_scan_bytes=total_input_bytes;
feather->total_scan_cnt=total_scan_cnt;
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_LEN], 0,FS_OP_SET,feather->total_scan_bytes);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_CNT], 0,FS_OP_SET,feather->total_scan_cnt);
FS_passive_output(feather->stat_handle);
return;
}

6
src/entry/Makefile
View File

@@ -6,7 +6,7 @@ CCC = g++
CFLAGS = -Wall -g -fPIC
CFLAGS += $(OPTFLAGS)
LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc
LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre
LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lrt -lm -lrulescan -lpcre -lMESA_field_stat2
#LDFLAGS += $(LDDICTATOR)
MAILLIB = ../lib
@@ -15,8 +15,8 @@ H_DIR =-I$(G_H_DIR) -I../../inc
LIBMAAT = libmaatframe.a
LIBMAAT_SO = libmaatframe.so
OBJS=config_monitor.o Maat_rule.o Maat_api.o UniversalBoolMatch.o dynamic_array.o cJSON.o json2iris.o map_str2int.o\
interval_index.o great_index_engine.o mesa_fuzzy.o
OBJS=config_monitor.o Maat_rule.o Maat_api.o Maat_stat.o UniversalBoolMatch.o dynamic_array.o cJSON.o\
json2iris.o map_str2int.o interval_index.o great_index_engine.o mesa_fuzzy.o rbtree.o
.c.o:
$(CC) -c $(CFLAGS) -I. $(H_DIR) $<

52
src/entry/aligment_int64.h Normal file
View File

@@ -0,0 +1,52 @@
#ifndef H_ALIGMENT_INT64_H_INCLUDE
#define H_ALIGMENT_INT64_H_INCLUDE
#include <stdlib.h>
#define CPU_CACHE_ALIGMENT 64
typedef long long* mcore_long_t;
inline long long *aligment_int64_array_alloc(int size)
{
long long *ret=NULL;
ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size);
return ret;
}
inline long long aligment_int64_array_sum(mcore_long_t array,int size)
{
long long sum=0;
int offset=0,i=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(long long))*i;
sum+=array[offset];
}
return sum;
}
inline long long aligment_int64_array_add(mcore_long_t array,int offset,long long op_val)
{
int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset;
array[idx]+=op_val;
return array[idx];
}
inline long long aligment_int64_array_cnt(mcore_long_t array,int size)
{
int offset=0,i=0;
int cnt=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(long long))*i;
if(array[offset]>0)
{
cnt++;
}
}
return cnt;
}
inline void aligment_int64_array_free(mcore_long_t array)
{
free(array);
}
#endif

4
src/entry/great_index_engine.c
View File

@@ -97,7 +97,7 @@ GIE_handle_t * GIE_create(const GIE_create_para_t * para)
idtable_args.hash_slot_size = HTABLE_SIZE;
idtable_args.max_elem_num = 4 * HTABLE_SIZE;
idtable_args.expire_time = 0;
idtable_args.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
idtable_args.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
idtable_args.key_comp = NULL;
idtable_args.key2index = NULL;
idtable_args.data_free = idtable_free;
@@ -108,7 +108,7 @@ GIE_handle_t * GIE_create(const GIE_create_para_t * para)
indextable_args.hash_slot_size = HTABLE_SIZE;
indextable_args.max_elem_num = 4 * HTABLE_SIZE;
indextable_args.expire_time = 0;
indextable_args.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
indextable_args.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
indextable_args.key_comp = NULL;
indextable_args.key2index = NULL;
indextable_args.data_free = indextable_free;

0
src/inc_internal/great_index_engine.h → src/entry/great_index_engine.h
View File

669
src/entry/interval_index.c
View File

@@ -1,7 +1,17 @@
#include<stdio.h>
#include<stdlib.h>
#include"interval_index.h"
/*********************************************************************
* File:
* interval_index.c
* Author:
* TangQi
* E-mail:
* tangqi@iie.ac.cn
*********************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include "interval_index.h"
#include "rbtree.h"
#include "rbtree_augmented.h"
/**
* There is a trick here. In order to hide specific
@@ -16,73 +26,113 @@
* Structure of inner segment
**/
typedef struct __IVI_shadow_seg_t{
IVI_seg_t lightseg;
TAILQ_ENTRY(__IVI_shadow_seg_t) ENTRY;
IVI_seg_t lightseg; /* interval for user, including left edge, right edge, and user's data */
struct rb_node rb; /* node of rb-tree */
OFFSET_TYPE max; /* max edge of subtree */
}IVI_shadow_seg_t;
TAILQ_HEAD(TQ, __IVI_shadow_seg_t);
/* Structure of inner InterVal Index */
typedef struct __IVI_shadow_t{
struct TQ ivi_queue;
struct rb_root root;
/* statistics */
int segs_cnt;
OFFSET_TYPE segs_length;
unsigned long long mem_occupy; //do not include user data
}IVI_shadow_t;
/**
* new is closer to head or tail ?
* Return 1 if closer to head than tail
* Else return 0
*/
int closer_to_head(IVI_shadow_seg_t * head, IVI_shadow_seg_t * tail, OFFSET_TYPE target)
IVI_seg_t * IVI_first_seg(IVI_t * handler)
{
if(head == NULL || tail == NULL)
return 1;
S_OFFSET_TYPE tmp1 = (S_OFFSET_TYPE)(target - head->lightseg.left);
S_OFFSET_TYPE tmp2 = (S_OFFSET_TYPE)(target - tail->lightseg.left);
S_OFFSET_TYPE distance_to_head = tmp1 > 0 ? tmp1 : -tmp1;
S_OFFSET_TYPE distance_to_tail = tmp2 > 0 ? tmp2 : -tmp2;
return (distance_to_tail - distance_to_head > 0);
assert(handler != NULL);
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
struct rb_node *first_node = rb_first(&(shadow_ivi->root));
if(first_node == NULL)
return NULL;
return (IVI_seg_t *)(rb_entry(first_node, IVI_shadow_seg_t, rb));
}
IVI_seg_t * IVI_last_seg(IVI_t * handler)
{
assert(handler != NULL);
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
struct rb_node *last_node = rb_last(&(shadow_ivi->root));
if(last_node == NULL)
return NULL;
return (IVI_seg_t *)(rb_entry(last_node, IVI_shadow_seg_t, rb));
}
IVI_seg_t * IVI_prev_seg(IVI_seg_t * seg)
{
assert(seg != NULL);
IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg;
struct rb_node * prev_node = rb_prev(&(shadow_seg->rb));
if(prev_node == NULL)
return NULL;
return (IVI_seg_t *)(rb_entry(prev_node, IVI_shadow_seg_t, rb));
}
IVI_seg_t * IVI_next_seg(IVI_seg_t * seg)
{
assert(seg != NULL);
IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg;
struct rb_node * next_node = rb_next(&(shadow_seg->rb));
if(next_node == NULL)
return NULL;
return (IVI_seg_t *)(rb_entry(next_node, IVI_shadow_seg_t, rb));
}
IVI_seg_t * IVI_prev_continuous_seg(IVI_seg_t * seg)
{
if(NULL == seg)
assert(seg != NULL);
IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg;
struct rb_node * prev_node = rb_prev(&(shadow_seg->rb));
if(prev_node == NULL)
{
return NULL;
}
IVI_shadow_seg_t * _seg = (IVI_shadow_seg_t *)seg;
IVI_shadow_seg_t * prev = TAILQ_PREV(_seg, TQ, ENTRY);
if(NULL == prev)
IVI_seg_t * prev_seg = (IVI_seg_t *)(rb_entry(prev_node, IVI_shadow_seg_t, rb));
if(!continuous(prev_seg->right, seg->left))
{
return NULL;
}
if(continuous((prev->lightseg).right, seg->left))
return (IVI_seg_t *)prev;
return NULL;
return prev_seg;
}
IVI_seg_t * IVI_next_continuous_seg(IVI_seg_t * seg)
{
if(NULL == seg)
assert(seg != NULL);
IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg;
struct rb_node * next_node = rb_next(&(shadow_seg->rb));
if(next_node == NULL)
{
return NULL;
}
IVI_shadow_seg_t * _seg = (IVI_shadow_seg_t *)seg;
IVI_shadow_seg_t * next = TAILQ_NEXT(_seg, ENTRY);
if(NULL == next)
IVI_seg_t * next_seg = (IVI_seg_t *)(rb_entry(next_node, IVI_shadow_seg_t, rb));
if(!continuous(seg->right, next_seg->left))
{
return NULL;
}
if(continuous(seg->right, (next->lightseg).left))
return (IVI_seg_t *)next;
return NULL;
return next_seg;
}
static inline int __is_overlapped(OFFSET_TYPE left1, OFFSET_TYPE right1, OFFSET_TYPE left2, OFFSET_TYPE right2)
{
if(!after(left1, right2) && !after(left2, right1))
return 1;
return 0;
}
@@ -152,13 +202,42 @@ Relation_t IVI_relative_position(IVI_seg_t * seg1, IVI_seg_t * seg2)
IVI_t * IVI_create(void)
{
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)malloc(sizeof(IVI_shadow_t));
TAILQ_INIT(&(shadow_ivi->ivi_queue));
shadow_ivi->root = RB_ROOT; //init rb tree's root
shadow_ivi->segs_cnt = 0;
shadow_ivi->segs_length = 0;
shadow_ivi->mem_occupy = sizeof(IVI_shadow_t);
return (IVI_t *)shadow_ivi;
}
static void __free_rb_tree(struct rb_node * root, IVI_callback_t cb, void * usr_para)
{
if(root == NULL)
{
return;
}
if(root->rb_left != NULL)
{
__free_rb_tree(root->rb_left, cb, usr_para);
}
if(root->rb_right != NULL)
{
__free_rb_tree(root->rb_right, cb, usr_para);
}
/* free user data */
IVI_shadow_seg_t * shadow_seg = rb_entry(root, IVI_shadow_seg_t, rb);
if(cb != NULL)
{
cb((IVI_seg_t *)shadow_seg, usr_para);
}
/* free seg */
free(shadow_seg);
shadow_seg = NULL;
return;
}
/**
* Name:
* IVI_destroy
@@ -177,26 +256,11 @@ void IVI_destroy(IVI_t * handler, IVI_callback_t cb, void * usr_para)
{
return;
}
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
IVI_shadow_seg_t * tmpseg = TAILQ_FIRST(&(shadow_ivi->ivi_queue));
IVI_shadow_seg_t * tmp;
/* Free each seg in IVI */
while(tmpseg != NULL)
{
tmp = TAILQ_NEXT(tmpseg, ENTRY);
/* Free *data in seg */
if(NULL != cb)
{
cb(&(tmpseg->lightseg), usr_para);
}
free(tmpseg);
tmpseg = tmp;
}
/* Free IVI */
__free_rb_tree(shadow_ivi->root.rb_node, cb, usr_para);
free(shadow_ivi);
handler = NULL;
return;
}
@@ -224,6 +288,7 @@ IVI_seg_t * IVI_seg_malloc(OFFSET_TYPE left, OFFSET_TYPE right, void * data)
shadow_seg->lightseg.left = left;
shadow_seg->lightseg.right= right;
shadow_seg->lightseg.data = data;
shadow_seg->max = 0;
return (IVI_seg_t *)shadow_seg;
}
@@ -244,6 +309,8 @@ IVI_seg_t * IVI_seg_malloc(OFFSET_TYPE left, OFFSET_TYPE right, void * data)
**/
void IVI_seg_free(IVI_seg_t * seg, IVI_callback_t cb, void * usr_para)
{
assert(seg != NULL);
/* Free user data first */
if(cb != NULL)
{
@@ -257,6 +324,70 @@ void IVI_seg_free(IVI_seg_t * seg, IVI_callback_t cb, void * usr_para)
}
static inline OFFSET_TYPE __interval_tree_get_subtree_max(IVI_shadow_seg_t * node)
{
OFFSET_TYPE max = node->lightseg.right, subtree_max;
if(node->rb.rb_left)
{
subtree_max = (rb_entry(node->rb.rb_left, IVI_shadow_seg_t, rb))->max;
if(before(max, subtree_max))
max = subtree_max;
}
if(node->rb.rb_right)
{
subtree_max = (rb_entry(node->rb.rb_right, IVI_shadow_seg_t, rb))->max;
if(before(max, subtree_max))
max = subtree_max;
}
return max;
}
static void __interval_tree_augment_propagate(struct rb_node * rb, struct rb_node * stop)
{
while(rb != stop)
{
IVI_shadow_seg_t * node = rb_entry(rb, IVI_shadow_seg_t, rb);
OFFSET_TYPE subtree_max = __interval_tree_get_subtree_max(node);
if(node->max == subtree_max)
{
break;
}
node->max = subtree_max;
rb = rb_parent(&node->rb);
}
return;
}
static void __interval_tree_augment_copy(struct rb_node * rb_old, struct rb_node * rb_new)
{
IVI_shadow_seg_t * old = rb_entry(rb_old, IVI_shadow_seg_t, rb);
IVI_shadow_seg_t * new = rb_entry(rb_new, IVI_shadow_seg_t, rb);
new->max = old->max;
return;
}
static void __interval_tree_augment_rotate(struct rb_node * rb_old, struct rb_node * rb_new)
{
IVI_shadow_seg_t * old = rb_entry(rb_old, IVI_shadow_seg_t, rb);
IVI_shadow_seg_t * new = rb_entry(rb_new, IVI_shadow_seg_t, rb);
new->max = old->max;
old->max = __interval_tree_get_subtree_max(old);
return;
}
static const struct rb_augment_callbacks __interval_tree_augment_callbacks = {
__interval_tree_augment_propagate,
__interval_tree_augment_copy,
__interval_tree_augment_rotate
};
/**
* Name:
* IVI_insert
@@ -273,74 +404,49 @@ void IVI_seg_free(IVI_seg_t * seg, IVI_callback_t cb, void * usr_para)
**/
int IVI_insert(IVI_t * handler, IVI_seg_t * seg)
{
IVI_shadow_t * shadow_ivi;
IVI_shadow_seg_t *head, *tail, *new_seg, *tmp_seg;
if(NULL == handler || NULL == seg)
{
return -1;
}
shadow_ivi = (IVI_shadow_t *)handler;
new_seg = (IVI_shadow_seg_t *)seg;
head = TAILQ_FIRST(&(shadow_ivi->ivi_queue));
tail = TAILQ_LAST(&(shadow_ivi->ivi_queue), TQ);
if(closer_to_head(head, tail, seg->left))
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
struct rb_root * root = &(shadow_ivi->root);
OFFSET_TYPE left = seg->left, right = seg->right;
struct rb_node **link = &root->rb_node, *rb_parent = NULL;
IVI_shadow_seg_t * parent = NULL;
IVI_shadow_seg_t * new_seg = (IVI_shadow_seg_t *)seg;
while(*link)
{
TAILQ_FOREACH(tmp_seg, &(shadow_ivi->ivi_queue), ENTRY)
rb_parent = *link;
parent = rb_entry(rb_parent, IVI_shadow_seg_t, rb);
/* is overlapped */
if(__is_overlapped(left, right, parent->lightseg.left, parent->lightseg.right))
{
//overlapped, return
return -1;
}
if(before(parent->max, right))
{
/* Find the first seg whose left is bigger than given seg's right, we will insert new seg before it */
if(after(tmp_seg->lightseg.left, new_seg->lightseg.right))
{
TAILQ_INSERT_BEFORE(tmp_seg, new_seg, ENTRY);
shadow_ivi->segs_cnt ++;
shadow_ivi->segs_length += (seg->right - seg->left + 1);
return 0;
}
else if(before(tmp_seg->lightseg.right, new_seg->lightseg.left))
{
continue;
}
else /* Overlap */
{
return -1;
}
parent->max = right;
}
/* If have searched to the end of list, we will inset it to the tail */
TAILQ_INSERT_TAIL(&(shadow_ivi->ivi_queue), new_seg, ENTRY);
shadow_ivi->segs_cnt ++;
shadow_ivi->segs_length += (seg->right - seg->left + 1);
}
else
{
TAILQ_FOREACH_REVERSE(tmp_seg, &(shadow_ivi->ivi_queue), TQ, ENTRY)
if(before(left, parent->lightseg.left))
{
/* Find the first seg whose right is smaller than given seg's left, we will insert new seg after it */
if(before(tmp_seg->lightseg.right, new_seg->lightseg.left))
{
TAILQ_INSERT_AFTER(&(shadow_ivi->ivi_queue), tmp_seg, new_seg, ENTRY);
shadow_ivi->segs_cnt ++;
shadow_ivi->segs_length += (seg->right - seg->left + 1);
return 0;
}
else if(after(tmp_seg->lightseg.left, new_seg->lightseg.right))
{
continue;
}
else /* Overlap */
{
return -1;
}
link = &parent->rb.rb_left;
}
else
{
link = &parent->rb.rb_right;
}
/* If have searched to the head of list, we will inset it to the head */
TAILQ_INSERT_HEAD(&(shadow_ivi->ivi_queue), new_seg, ENTRY);
shadow_ivi->segs_cnt ++;
shadow_ivi->segs_length += (seg->right - seg->left + 1);
}
new_seg->max = right;
rb_link_node(&new_seg->rb, rb_parent, link);
rb_insert_augmented(&new_seg->rb, root, &__interval_tree_augment_callbacks);
/* updata statistics */
shadow_ivi->segs_cnt ++;
shadow_ivi->segs_length += seg->right - seg->left + 1;
shadow_ivi->mem_occupy += sizeof(IVI_shadow_seg_t);
return 0;
}
@@ -367,16 +473,56 @@ int IVI_remove(IVI_t * handler, IVI_seg_t * seg)
}
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg;
struct rb_root * root = &(shadow_ivi->root);
IVI_shadow_seg_t * new_seg = (IVI_shadow_seg_t *)seg;
rb_erase_augmented(&new_seg->rb, root, &__interval_tree_augment_callbacks);
TAILQ_REMOVE(&(shadow_ivi->ivi_queue), shadow_seg, ENTRY);
/* updata statistics */
shadow_ivi->segs_cnt --;
shadow_ivi->segs_length -= (seg->right - seg->left + 1);
shadow_ivi->segs_length -= seg->right - seg->left + 1;
shadow_ivi->mem_occupy -= sizeof(IVI_shadow_seg_t);
return 0;
}
static struct rb_node * __min_interval_search_from(struct rb_node * node, OFFSET_TYPE left, OFFSET_TYPE right)
{
if(node == NULL)
{
return NULL;
}
IVI_shadow_seg_t * seg = rb_entry(node, IVI_shadow_seg_t, rb);
IVI_shadow_seg_t * left_seg = rb_entry(node->rb_left, IVI_shadow_seg_t, rb);
if(node->rb_left != NULL && !before(left_seg->max, left))
{
struct rb_node * ret = __min_interval_search_from(node->rb_left, left, right);
if(ret != NULL)
{
return ret;
}
else if(__is_overlapped(left, right, seg->lightseg.left, seg->lightseg.right))
{
return node;
}
else
{
return NULL;
}
}
else if(__is_overlapped(left, right, seg->lightseg.left, seg->lightseg.right))
{
return node;
}
else
{
return __min_interval_search_from(node->rb_right, left, right);
}
}
/**
* Name:
* IVI_query
@@ -397,115 +543,34 @@ int IVI_remove(IVI_t * handler, IVI_seg_t * seg)
**/
int IVI_query(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs)
{
IVI_shadow_t * shadow_ivi;
IVI_shadow_seg_t *head, *tail, *tmp, *left_tmp, *right_tmp;
int interval_cnt = 0, i;
if(NULL == handler || after(left, right))
{
//augments error
return -1;
}
shadow_ivi = (IVI_shadow_t *)handler;
head = TAILQ_FIRST(&(shadow_ivi->ivi_queue));
tail = TAILQ_LAST(&(shadow_ivi->ivi_queue), TQ);
int interval_cnt = 0, max_cnt = 8;
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
struct rb_node * root = shadow_ivi->root.rb_node;
struct rb_node * min_overlap = __min_interval_search_from(root, left, right);
struct rb_node * tmp_node = min_overlap;
/* Traverse from head or tail? We need to decide */
if(closer_to_head(head, tail, left))
*segs = (IVI_seg_t **)malloc(max_cnt * sizeof(IVI_seg_t *));
while (tmp_node != NULL)
{
tmp = head;
while(tmp != NULL)
IVI_seg_t * tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb));
if(!__is_overlapped(tmp_seg->left, tmp_seg->right, left, right))
{
if(after(left, tmp->lightseg.right))
{
tmp = TAILQ_NEXT(tmp, ENTRY);
}
else
{
/* Get the seg which left is in or before*/
left_tmp = tmp;
break;
}
break;
}
if(tmp == NULL)
if(interval_cnt > max_cnt)
{
*segs = NULL;
return 0;
}
/* Get the num of overlapped segs */
while(tmp != NULL)
{
if(!before(right, tmp->lightseg.left))
{
tmp = TAILQ_NEXT(tmp, ENTRY);
interval_cnt ++;
}
else
{
break;
}
}
tmp = left_tmp;
if(interval_cnt == 0)
{
*segs = NULL;
return 0;
}
*segs = (IVI_seg_t **)malloc(interval_cnt * sizeof(IVI_seg_t *));
for(i = 0; i < interval_cnt; i++)
{
(*segs)[i] = (IVI_seg_t *)tmp;
tmp = TAILQ_NEXT(tmp, ENTRY);
}
}
else
{
tmp = tail;
while(tmp != NULL)
{
if(before(right, tmp->lightseg.left))
{
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
}
else
{
right_tmp = tmp;
break;
}
}
if(tmp == NULL)
{
*segs = NULL;
return 0;
}
/* Get the num of overlapped segs */
while(tmp != NULL)
{
if(!after(left, tmp->lightseg.right))
{
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
interval_cnt ++;
}
else
{
break;
}
}
tmp = right_tmp;
if(interval_cnt == 0)
{
*segs = NULL;
return 0;
}
*segs = (IVI_seg_t **)malloc(interval_cnt * sizeof(IVI_seg_t *));
for(i = interval_cnt - 1; i >= 0; i--)
{
(*segs)[i] = (IVI_seg_t *)tmp;
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
max_cnt *= 2;
*segs = (IVI_seg_t **)realloc(*segs, max_cnt * sizeof(IVI_seg_t *));
}
(*segs)[interval_cnt] = tmp_seg;
interval_cnt ++;
tmp_node = rb_next(tmp_node);
}
return interval_cnt;
}
@@ -531,129 +596,41 @@ int IVI_query(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t **
**/
int IVI_query_continuous(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs)
{
IVI_shadow_t * shadow_ivi;
IVI_shadow_seg_t *head, *tail, *tmp, *left_tmp, *right_tmp;
int interval_cnt = 0, i;
if(NULL == handler || after(left, right))
{
//augments error
return -1;
}
shadow_ivi = (IVI_shadow_t *)handler;
head = TAILQ_FIRST(&(shadow_ivi->ivi_queue));
tail = TAILQ_LAST(&(shadow_ivi->ivi_queue), TQ);
int interval_cnt = 0, max_cnt = 8;
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
struct rb_node * root = shadow_ivi->root.rb_node;
struct rb_node * min_overlap = __min_interval_search_from(root, left, right);
struct rb_node * tmp_node = min_overlap;
/* Traverse from head or tail? We need to decide */
if(closer_to_head(head, tail, left))
*segs = (IVI_seg_t **)malloc(max_cnt * sizeof(IVI_seg_t *));
while (tmp_node != NULL)
{
tmp = head;
while(tmp != NULL)
IVI_seg_t * tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb));
if(!__is_overlapped(tmp_seg->left, tmp_seg->right, left, right))
{
if(after(left, tmp->lightseg.right))
{
tmp = TAILQ_NEXT(tmp, ENTRY);
}
else
{
/* Get the seg which left is in or before*/
left_tmp = tmp;
break;
}
break;
}
if(tmp == NULL)
if(interval_cnt > max_cnt)
{
*segs = NULL;
return 0;
max_cnt += 8;
*segs = (IVI_seg_t **)realloc(*segs, max_cnt * sizeof(IVI_seg_t *));
}
/* Get the num of overlapped segs */
while(tmp != NULL)
(*segs)[interval_cnt] = tmp_seg;
interval_cnt ++;
tmp_node = rb_next(tmp_node);
IVI_seg_t * prev_tmp_seg = tmp_seg;
tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb));
if(!continuous(prev_tmp_seg->right, tmp_seg->left))
{
if(!before(right, tmp->lightseg.left))
{
tmp = TAILQ_NEXT(tmp, ENTRY);
interval_cnt ++;
}
else
{
break;
}
IVI_shadow_seg_t * prev = TAILQ_PREV(tmp, TQ, ENTRY);
if(tmp != NULL && !continuous(prev->lightseg.right, tmp->lightseg.left))
{
break;
}
}
tmp = left_tmp;
if(interval_cnt == 0)
{
*segs = NULL;
return 0;
}
*segs = (IVI_seg_t **)malloc(interval_cnt * sizeof(IVI_seg_t *));
for(i = 0; i < interval_cnt; i++)
{
(*segs)[i] = (IVI_seg_t *)tmp;
tmp = TAILQ_NEXT(tmp, ENTRY);
break;
}
}
else
{
tmp = tail;
while(tmp != NULL)
{
if(before(right, tmp->lightseg.left))
{
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
}
else
{
right_tmp = tmp;
break;
}
}
if(tmp == NULL)
{
*segs = NULL;
return 0;
}
/* Get the num of overlapped segs */
while(tmp != NULL)
{
if(!after(left, tmp->lightseg.right))
{
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
interval_cnt ++;
}
else
{
break;
}
IVI_shadow_seg_t * next = TAILQ_NEXT(tmp, ENTRY);
if(tmp != NULL && !continuous(tmp->lightseg.right, next->lightseg.left))
{
break;
}
}
tmp = right_tmp;
if(interval_cnt == 0)
{
*segs = NULL;
return 0;
}
*segs = (IVI_seg_t **)malloc(interval_cnt * sizeof(IVI_seg_t *));
for(i = interval_cnt - 1; i >= 0; i--)
{
(*segs)[i] = (IVI_seg_t *)tmp;
tmp = TAILQ_PREV(tmp, TQ, ENTRY);
}
}
return interval_cnt;
}
@@ -698,6 +675,41 @@ OFFSET_TYPE IVI_seg_length(IVI_t * handler)
}
/**
* Name:
* IVI_mem_occupy
* Description:
* Get the memory occupy of given interval index handler
* Params:
* handler: The handler of InterVal Index created by IVI_create.
* Return:
* Return the memory occupy of given interval index handler
**/
unsigned long long IVI_mem_occupy(IVI_t * handler)
{
if(handler == NULL)
return 0;
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
return shadow_ivi->mem_occupy;
}
static void __inorder_traverse(struct rb_node * root, IVI_callback_t cb, void * usr_para)
{
if(root == NULL)
{
return;
}
/* save first in case of root is freed in callback */
struct rb_node * left_node = root->rb_left;
struct rb_node * right_node = root->rb_right;
__inorder_traverse(left_node, cb, usr_para);
IVI_seg_t * seg = (IVI_seg_t *)(rb_entry(root, IVI_shadow_seg_t, rb));
cb(seg, usr_para);
__inorder_traverse(right_node, cb, usr_para);
return;
}
/**
* Name:
@@ -720,17 +732,6 @@ void IVI_traverse(IVI_t * handler, IVI_callback_t cb, void * usr_para)
}
IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler;
IVI_shadow_seg_t * tmp_seg = TAILQ_FIRST(&(shadow_ivi->ivi_queue));
IVI_shadow_seg_t * tmp;
/* Traverse the IVI */
while(tmp_seg != NULL)
{
/*
* The place we can't use TAILQ_FOREACH because we
* do not no what will callback funciton does.
* */
tmp = TAILQ_NEXT(tmp_seg, ENTRY);
cb((IVI_seg_t *)tmp_seg, usr_para);
tmp_seg = tmp;
}
__inorder_traverse(shadow_ivi->root.rb_node, cb, usr_para);
return;
}

31
src/inc_internal/interval_index.h → src/entry/interval_index.h
View File

@@ -6,7 +6,7 @@
* (3) The interval supports rollback.
*
* author: zhengchao@iie.ac.cn tangqi@iie.ac.cn
* last modify time: 2015-08-29
* last modify time: 2015-12-04
*************************************************************************/
#ifndef _INTERVAL_INDEX_H_
@@ -16,12 +16,11 @@
extern "C"{
#endif
#include "queue.h"
#define SIZE_8
#ifdef SIZE_8
typedef unsigned long long OFFSET_TYPE;
typedef unsigned long long OFFSET_TYPE;
typedef signed long long S_OFFSET_TYPE;
#else
typedef unsigned int OFFSET_TYPE;
@@ -61,6 +60,10 @@ static inline int continuous(OFFSET_TYPE prev, OFFSET_TYPE next)
}
IVI_seg_t * IVI_first_seg(IVI_t * handler);
IVI_seg_t * IVI_last_seg(IVI_t * handler);
IVI_seg_t * IVI_prev_seg(IVI_seg_t * seg);
IVI_seg_t * IVI_next_seg(IVI_seg_t * seg);
IVI_seg_t * IVI_prev_continuous_seg(IVI_seg_t * seg);
IVI_seg_t * IVI_next_continuous_seg(IVI_seg_t * seg);
@@ -71,22 +74,22 @@ IVI_seg_t * IVI_next_continuous_seg(IVI_seg_t * seg);
typedef enum __Relation_t{
LEFT_NO_OVERLAP = 1, // |___A___|
// |___B___|
LEFT_OVERLAP, // |___A___|
// |___B___|
CONTAINED, // |___A___|
// |_____B_____|
CONTAIN, // |_____A_____|
// |___B___|
RIGHT_OVERLAP, // |___A___|
// |___B___|
RIGHT_NO_OVERLAP, // |___A___|
// |___B___|
ERROR
}Relation_t;
@@ -273,6 +276,18 @@ int IVI_seg_cnt(IVI_t * handler);
OFFSET_TYPE IVI_seg_length(IVI_t * handler);
/**
* Name:
* IVI_mem_occupy
* Description:
* Get the memory occupy of given interval index handler
* Params:
* handler: The handler of InterVal Index created by IVI_create.
* Return:
* Return the memory occupy of given interval index handler
**/
unsigned long long IVI_mem_occupy(IVI_t * handler);
/**
* Name:

17
src/entry/json2iris.cpp
View File

@@ -89,7 +89,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_descriptio
hargs.thread_safe=1;
hargs.hash_slot_size = 1024;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL;
@@ -110,8 +110,11 @@ int set_iris_descriptor(const char* json_file,cJSON *json,struct iris_descriptio
map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP);
map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS);
map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTVAL);
map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST);
map_register(iris_cfg->str2int_map, "ipv4",4);
map_register(iris_cfg->str2int_map, "ipv6",6);
@@ -364,7 +367,7 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha
return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
}
int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger)
{
struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
int cmd_cnt=0;
@@ -378,6 +381,13 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
if(table_type==TABLE_TYPE_EXPR_PLUS)
{
json_cmd[cmd_cnt].json_string="district";
json_cmd[cmd_cnt].json_type=cJSON_String;
cmd_cnt++;
}
json_cmd[cmd_cnt].json_string="keywords";
json_cmd[cmd_cnt].json_type=cJSON_String;
cmd_cnt++;
@@ -598,7 +608,8 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
switch(table_type)
{
case TABLE_TYPE_EXPR:
ret=write_expr_rule(table_content, p_iris, table_info->table_path, logger);
case TABLE_TYPE_EXPR_PLUS:
ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger);
break;
case TABLE_TYPE_IP:
ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger);

38
src/entry/map_str2int.cpp
View File

@@ -1,21 +1,35 @@
#include <MESA/MESA_htable.h>
void map_tmp_free(void* ptr)
{
free(ptr);
}
long read_map_val(void *data, const uchar *key, uint size, void *user_arg)
{
if(data!=NULL)
{
*(int*)user_arg=*(int*)data;
return 1;
}
else
{
return 0;
}
}
MESA_htable_handle map_create(void)
{
MESA_htable_handle string2int_map;
MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=1;
hargs.hash_slot_size = 1024*1024;
hargs.thread_safe=8;
hargs.hash_slot_size = 4*1024;
hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_LRU;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
hargs.expire_time = 0;
hargs.key_comp = NULL;
hargs.key2index = NULL;
hargs.recursive = 0;
hargs.recursive = 1;
hargs.data_free = map_tmp_free;
hargs.data_expire_with_condition = NULL;
string2int_map=MESA_htable_create(&hargs, sizeof(hargs));
@@ -31,20 +45,28 @@ int map_register(MESA_htable_handle handle,const char* string,int value)
{
unsigned int size=strlen(string);
unsigned char *key=(unsigned char *)string;
int *data=(int*)malloc(sizeof(int));
int ret=0;
int * data= (int*)malloc(sizeof(int));
*data=value;
ret=MESA_htable_add(handle,key,size,data);
if(ret<0)
{
free(data);
}
return ret;
}
int map_str2int(MESA_htable_handle handle,const char* string,int* value)
{
int *data=NULL;
long cb_ret=0;
unsigned int size=strlen(string);
data=(int*)MESA_htable_search(handle,(unsigned char*)string,size);
if(data!=NULL)
data=(int*)MESA_htable_search_cb(handle,(unsigned char*)string,size,
read_map_val,value,&cb_ret);
// data=(int*)MESA_htable_search(handle,(unsigned char*)string,size);
if(cb_ret>0)
{
*value=*data;
return 1;
}
else

1267
src/entry/mesa_fuzzy.c
View File

File diff suppressed because it is too large Load Diff

548
src/entry/rbtree.c Normal file
View File

@@ -0,0 +1,548 @@
/*
Red Black Trees
(C) 1999 Andrea Arcangeli <andrea@suse.de>
(C) 2002 David Woodhouse <dwmw2@infradead.org>
(C) 2012 Michel Lespinasse <walken@google.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
linux/lib/rbtree.c
*/
#include "rbtree.h"
#include "rbtree_augmented.h"
/*
* red-black trees properties: http://en.wikipedia.org/wiki/Rbtree
*
* 1) A node is either red or black
* 2) The root is black
* 3) All leaves (NULL) are black
* 4) Both children of every red node are black
* 5) Every simple path from root to leaves contains the same number
* of black nodes.
*
* 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two
* consecutive red nodes in a path and every red node is therefore followed by
* a black. So if B is the number of black nodes on every simple path (as per
* 5), then the longest possible path due to 4 is 2B.
*
* We shall indicate color with case, where black nodes are uppercase and red
* nodes will be lowercase. Unknown color nodes shall be drawn as red within
* parentheses and have some accompanying text comment.
*/
static inline void rb_set_black(struct rb_node *rb)
{
rb->__rb_parent_color |= RB_BLACK;
}
static inline struct rb_node *rb_red_parent(struct rb_node *red)
{
return (struct rb_node *)red->__rb_parent_color;
}
/*
* Helper function for rotations:
* - old's parent and color get assigned to new
* - old gets assigned new as a parent and 'color' as a color.
*/
static inline void
__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new,
struct rb_root *root, int color)
{
struct rb_node *parent = rb_parent(old);
new->__rb_parent_color = old->__rb_parent_color;
rb_set_parent_color(old, new, color);
__rb_change_child(old, new, parent, root);
}
static __always_inline void
__rb_insert(struct rb_node *node, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
struct rb_node *parent = rb_red_parent(node), *gparent, *tmp;
while (1) {
/*
* Loop invariant: node is red
*
* If there is a black parent, we are done.
* Otherwise, take some corrective action as we don't
* want a red root or two consecutive red nodes.
*/
if (!parent) {
rb_set_parent_color(node, NULL, RB_BLACK);
break;
} else if (rb_is_black(parent))
break;
gparent = rb_red_parent(parent);
tmp = gparent->rb_right;
if (parent != tmp) { /* parent == gparent->rb_left */
if (tmp && rb_is_red(tmp)) {
/*
* Case 1 - color flips
*
* G g
* / \ / \
* p u --> P U
* / /
* n n
*
* However, since g's parent might be red, and
* 4) does not allow this, we need to recurse
* at g.
*/
rb_set_parent_color(tmp, gparent, RB_BLACK);
rb_set_parent_color(parent, gparent, RB_BLACK);
node = gparent;
parent = rb_parent(node);
rb_set_parent_color(node, parent, RB_RED);
continue;
}
tmp = parent->rb_right;
if (node == tmp) {
/*
* Case 2 - left rotate at parent
*
* G G
* / \ / \
* p U --> n U
* \ /
* n p
*
* This still leaves us in violation of 4), the
* continuation into Case 3 will fix that.
*/
parent->rb_right = tmp = node->rb_left;
node->rb_left = parent;
if (tmp)
rb_set_parent_color(tmp, parent,
RB_BLACK);
rb_set_parent_color(parent, node, RB_RED);
augment_rotate(parent, node);
parent = node;
tmp = node->rb_right;
}
/*
* Case 3 - right rotate at gparent
*
* G P
* / \ / \
* p U --> n g
* / \
* n U
*/
gparent->rb_left = tmp; /* == parent->rb_right */
parent->rb_right = gparent;
if (tmp)
rb_set_parent_color(tmp, gparent, RB_BLACK);
__rb_rotate_set_parents(gparent, parent, root, RB_RED);
augment_rotate(gparent, parent);
break;
} else {
tmp = gparent->rb_left;
if (tmp && rb_is_red(tmp)) {
/* Case 1 - color flips */
rb_set_parent_color(tmp, gparent, RB_BLACK);
rb_set_parent_color(parent, gparent, RB_BLACK);
node = gparent;
parent = rb_parent(node);
rb_set_parent_color(node, parent, RB_RED);
continue;
}
tmp = parent->rb_left;
if (node == tmp) {
/* Case 2 - right rotate at parent */
parent->rb_left = tmp = node->rb_right;
node->rb_right = parent;
if (tmp)
rb_set_parent_color(tmp, parent,
RB_BLACK);
rb_set_parent_color(parent, node, RB_RED);
augment_rotate(parent, node);
parent = node;
tmp = node->rb_left;
}
/* Case 3 - left rotate at gparent */
gparent->rb_right = tmp; /* == parent->rb_left */
parent->rb_left = gparent;
if (tmp)
rb_set_parent_color(tmp, gparent, RB_BLACK);
__rb_rotate_set_parents(gparent, parent, root, RB_RED);
augment_rotate(gparent, parent);
break;
}
}
}
/*
* Inline version for rb_erase() use - we want to be able to inline
* and eliminate the dummy_rotate callback there
*/
static __always_inline void
____rb_erase_color(struct rb_node *parent, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
struct rb_node *node = NULL, *sibling, *tmp1, *tmp2;
while (1) {
/*
* Loop invariants:
* - node is black (or NULL on first iteration)
* - node is not the root (parent is not NULL)
* - All leaf paths going through parent and node have a
* black node count that is 1 lower than other leaf paths.
*/
sibling = parent->rb_right;
if (node != sibling) { /* node == parent->rb_left */
if (rb_is_red(sibling)) {
/*
* Case 1 - left rotate at parent
*
* P S
* / \ / \
* N s --> p Sr
* / \ / \
* Sl Sr N Sl
*/
parent->rb_right = tmp1 = sibling->rb_left;
sibling->rb_left = parent;
rb_set_parent_color(tmp1, parent, RB_BLACK);
__rb_rotate_set_parents(parent, sibling, root,
RB_RED);
augment_rotate(parent, sibling);
sibling = tmp1;
}
tmp1 = sibling->rb_right;
if (!tmp1 || rb_is_black(tmp1)) {
tmp2 = sibling->rb_left;
if (!tmp2 || rb_is_black(tmp2)) {
/*
* Case 2 - sibling color flip
* (p could be either color here)
*
* (p) (p)
* / \ / \
* N S --> N s
* / \ / \
* Sl Sr Sl Sr
*
* This leaves us violating 5) which
* can be fixed by flipping p to black
* if it was red, or by recursing at p.
* p is red when coming from Case 1.
*/
rb_set_parent_color(sibling, parent,
RB_RED);
if (rb_is_red(parent))
rb_set_black(parent);
else {
node = parent;
parent = rb_parent(node);
if (parent)
continue;
}
break;
}
/*
* Case 3 - right rotate at sibling
* (p could be either color here)
*
* (p) (p)
* / \ / \
* N S --> N Sl
* / \ \
* sl Sr s
* \
* Sr
*/
sibling->rb_left = tmp1 = tmp2->rb_right;
tmp2->rb_right = sibling;
parent->rb_right = tmp2;
if (tmp1)
rb_set_parent_color(tmp1, sibling,
RB_BLACK);
augment_rotate(sibling, tmp2);
tmp1 = sibling;
sibling = tmp2;
}
/*
* Case 4 - left rotate at parent + color flips
* (p and sl could be either color here.
* After rotation, p becomes black, s acquires
* p's color, and sl keeps its color)
*
* (p) (s)
* / \ / \
* N S --> P Sr
* / \ / \
* (sl) sr N (sl)
*/
parent->rb_right = tmp2 = sibling->rb_left;
sibling->rb_left = parent;
rb_set_parent_color(tmp1, sibling, RB_BLACK);
if (tmp2)
rb_set_parent(tmp2, parent);
__rb_rotate_set_parents(parent, sibling, root,
RB_BLACK);
augment_rotate(parent, sibling);
break;
} else {
sibling = parent->rb_left;
if (rb_is_red(sibling)) {
/* Case 1 - right rotate at parent */
parent->rb_left = tmp1 = sibling->rb_right;
sibling->rb_right = parent;
rb_set_parent_color(tmp1, parent, RB_BLACK);
__rb_rotate_set_parents(parent, sibling, root,
RB_RED);
augment_rotate(parent, sibling);
sibling = tmp1;
}
tmp1 = sibling->rb_left;
if (!tmp1 || rb_is_black(tmp1)) {
tmp2 = sibling->rb_right;
if (!tmp2 || rb_is_black(tmp2)) {
/* Case 2 - sibling color flip */
rb_set_parent_color(sibling, parent,
RB_RED);
if (rb_is_red(parent))
rb_set_black(parent);
else {
node = parent;
parent = rb_parent(node);
if (parent)
continue;
}
break;
}
/* Case 3 - right rotate at sibling */
sibling->rb_right = tmp1 = tmp2->rb_left;
tmp2->rb_left = sibling;
parent->rb_left = tmp2;
if (tmp1)
rb_set_parent_color(tmp1, sibling,
RB_BLACK);
augment_rotate(sibling, tmp2);
tmp1 = sibling;
sibling = tmp2;
}
/* Case 4 - left rotate at parent + color flips */
parent->rb_left = tmp2 = sibling->rb_right;
sibling->rb_right = parent;
rb_set_parent_color(tmp1, sibling, RB_BLACK);
if (tmp2)
rb_set_parent(tmp2, parent);
__rb_rotate_set_parents(parent, sibling, root,
RB_BLACK);
augment_rotate(parent, sibling);
break;
}
}
}
/* Non-inline version for rb_erase_augmented() use */
void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
____rb_erase_color(parent, root, augment_rotate);
}
/*
* Non-augmented rbtree manipulation functions.
*
* We use dummy augmented callbacks here, and have the compiler optimize them
* out of the rb_insert_color() and rb_erase() function definitions.
*/
static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {}
static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {}
static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {}
static const struct rb_augment_callbacks dummy_callbacks = {
dummy_propagate, dummy_copy, dummy_rotate
};
void rb_insert_color(struct rb_node *node, struct rb_root *root)
{
__rb_insert(node, root, dummy_rotate);
}
void rb_erase(struct rb_node *node, struct rb_root *root)
{
struct rb_node *rebalance;
rebalance = __rb_erase_augmented(node, root, &dummy_callbacks);
if (rebalance)
____rb_erase_color(rebalance, root, dummy_rotate);
}
/*
* Augmented rbtree manipulation functions.
*
* This instantiates the same __always_inline functions as in the non-augmented
* case, but this time with user-defined callbacks.
*/
void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new))
{
__rb_insert(node, root, augment_rotate);
}
/*
* This function returns the first node (in sort order) of the tree.
*/
struct rb_node *rb_first(const struct rb_root *root)
{
struct rb_node *n;
n = root->rb_node;
if (!n)
return NULL;
while (n->rb_left)
n = n->rb_left;
return n;
}
struct rb_node *rb_last(const struct rb_root *root)
{
struct rb_node *n;
n = root->rb_node;
if (!n)
return NULL;
while (n->rb_right)
n = n->rb_right;
return n;
}
struct rb_node *rb_next(const struct rb_node *node)
{
struct rb_node *parent;
if (RB_EMPTY_NODE(node))
return NULL;
/*
* If we have a right-hand child, go down and then left as far
* as we can.
*/
if (node->rb_right) {
node = node->rb_right;
while (node->rb_left)
node=node->rb_left;
return (struct rb_node *)node;
}
/*
* No right-hand children. Everything down and left is smaller than us,
* so any 'next' node must be in the general direction of our parent.
* Go up the tree; any time the ancestor is a right-hand child of its
* parent, keep going up. First time it's a left-hand child of its
* parent, said parent is our 'next' node.
*/
while ((parent = rb_parent(node)) && node == parent->rb_right)
node = parent;
return parent;
}
struct rb_node *rb_prev(const struct rb_node *node)
{
struct rb_node *parent;
if (RB_EMPTY_NODE(node))
return NULL;
/*
* If we have a left-hand child, go down and then right as far
* as we can.
*/
if (node->rb_left) {
node = node->rb_left;
while (node->rb_right)
node=node->rb_right;
return (struct rb_node *)node;
}
/*
* No left-hand children. Go up till we find an ancestor which
* is a right-hand child of its parent.
*/
while ((parent = rb_parent(node)) && node == parent->rb_left)
node = parent;
return parent;
}
void rb_replace_node(struct rb_node *victim, struct rb_node *new,
struct rb_root *root)
{
struct rb_node *parent = rb_parent(victim);
/* Set the surrounding nodes to point to the replacement */
__rb_change_child(victim, new, parent, root);
if (victim->rb_left)
rb_set_parent(victim->rb_left, new);
if (victim->rb_right)
rb_set_parent(victim->rb_right, new);
/* Copy the pointers/colour from the victim to the replacement */
*new = *victim;
}
static struct rb_node *rb_left_deepest_node(const struct rb_node *node)
{
for (;;) {
if (node->rb_left)
node = node->rb_left;
else if (node->rb_right)
node = node->rb_right;
else
return (struct rb_node *)node;
}
}
struct rb_node *rb_next_postorder(const struct rb_node *node)
{
const struct rb_node *parent;
if (!node)
return NULL;
parent = rb_parent(node);
/* If we're sitting on node, we've already seen our children */
if (parent && node == parent->rb_left && parent->rb_right) {
/* If we are the parent's left node, go to the parent's right
* node then all the way down to the left */
return rb_left_deepest_node(parent->rb_right);
} else
/* Otherwise we are the parent's right node, and the parent
* should be next */
return (struct rb_node *)parent;
}
struct rb_node *rb_first_postorder(const struct rb_root *root)
{
if (!root->rb_node)
return NULL;
return rb_left_deepest_node(root->rb_node);
}

118
src/entry/rbtree.h Normal file
View File

@@ -0,0 +1,118 @@
/*
Red Black Trees
(C) 1999 Andrea Arcangeli <andrea@suse.de>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
linux/include/linux/rbtree.h
To use rbtrees you'll have to implement your own insert and search cores.
This will avoid us to use callbacks and to drop drammatically performances.
I know it's not the cleaner way, but in C (not in C++) to get
performances and genericity...
See Documentation/rbtree.txt for documentation and samples.
*/
#ifndef _LINUX_RBTREE_H
#define _LINUX_RBTREE_H
#include <stdio.h>
struct rb_node {
unsigned long __rb_parent_color;
struct rb_node *rb_right;
struct rb_node *rb_left;
} __attribute__((aligned(sizeof(long))));
/* The alignment might seem pointless, but allegedly CRIS needs it */
struct rb_root {
struct rb_node *rb_node;
};
#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3))
#define RB_ROOT (struct rb_root) { NULL, }
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type, member) );})
#endif
#define rb_entry(ptr, type, member) container_of(ptr, type, member)
#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL)
/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */
#define RB_EMPTY_NODE(node) \
((node)->__rb_parent_color == (unsigned long)(node))
#define RB_CLEAR_NODE(node) \
((node)->__rb_parent_color = (unsigned long)(node))
extern void rb_insert_color(struct rb_node *, struct rb_root *);
extern void rb_erase(struct rb_node *, struct rb_root *);
/* Find logical next and previous nodes in a tree */
extern struct rb_node *rb_next(const struct rb_node *);
extern struct rb_node *rb_prev(const struct rb_node *);
extern struct rb_node *rb_first(const struct rb_root *);
extern struct rb_node *rb_last(const struct rb_root *);
/* Postorder iteration - always visit the parent after its children */
extern struct rb_node *rb_first_postorder(const struct rb_root *);
extern struct rb_node *rb_next_postorder(const struct rb_node *);
/* Fast replacement of a single node without remove/rebalance/add/rebalance */
extern void rb_replace_node(struct rb_node *victim, struct rb_node *new,
struct rb_root *root);
static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
struct rb_node ** rb_link)
{
node->__rb_parent_color = (unsigned long)parent;
node->rb_left = node->rb_right = NULL;
*rb_link = node;
}
#define rb_entry_safe(ptr, type, member) \
({ typeof(ptr) ____ptr = (ptr); \
____ptr ? rb_entry(____ptr, type, member) : NULL; \
})
/**
* rbtree_postorder_for_each_entry_safe - iterate over rb_root in post order of
* given type safe against removal of rb_node entry
*
* @pos: the 'type *' to use as a loop cursor.
* @n: another 'type *' to use as temporary storage
* @root: 'rb_root *' of the rbtree.
* @field: the name of the rb_node field within 'type'.
*/
#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \
for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \
pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \
typeof(*pos), field); 1; }); \
pos = n)
#endif /* _LINUX_RBTREE_H */

241
src/entry/rbtree_augmented.h Normal file
View File

@@ -0,0 +1,241 @@
/*
Red Black Trees
(C) 1999 Andrea Arcangeli <andrea@suse.de>
(C) 2002 David Woodhouse <dwmw2@infradead.org>
(C) 2012 Michel Lespinasse <walken@google.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
linux/include/linux/rbtree_augmented.h
*/
#ifndef _LINUX_RBTREE_AUGMENTED_H
#define _LINUX_RBTREE_AUGMENTED_H
#include "rbtree.h"
/*
* Please note - only struct rb_augment_callbacks and the prototypes for
* rb_insert_augmented() and rb_erase_augmented() are intended to be public.
* The rest are implementation details you are not expected to depend on.
*
* See Documentation/rbtree.txt for documentation and samples.
*/
struct rb_augment_callbacks {
void (*propagate)(struct rb_node *node, struct rb_node *stop);
void (*copy)(struct rb_node *old, struct rb_node *new);
void (*rotate)(struct rb_node *old, struct rb_node *new);
};
extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
/*
* Fixup the rbtree and update the augmented information when rebalancing.
*
* On insertion, the user must update the augmented information on the path
* leading to the inserted node, then call rb_link_node() as usual and
* rb_augment_inserted() instead of the usual rb_insert_color() call.
* If rb_augment_inserted() rebalances the rbtree, it will callback into
* a user provided function to update the augmented information on the
* affected subtrees.
*/
static inline void
rb_insert_augmented(struct rb_node *node, struct rb_root *root,
const struct rb_augment_callbacks *augment)
{
__rb_insert_augmented(node, root, augment->rotate);
}
#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \
rbtype, rbaugmented, rbcompute) \
static inline void \
rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \
{ \
while (rb != stop) { \
rbstruct *node = rb_entry(rb, rbstruct, rbfield); \
rbtype augmented = rbcompute(node); \
if (node->rbaugmented == augmented) \
break; \
node->rbaugmented = augmented; \
rb = rb_parent(&node->rbfield); \
} \
} \
static inline void \
rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \
{ \
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
new->rbaugmented = old->rbaugmented; \
} \
static void \
rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \
{ \
rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \
rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \
new->rbaugmented = old->rbaugmented; \
old->rbaugmented = rbcompute(old); \
} \
rbstatic const struct rb_augment_callbacks rbname = { \
rbname ## _propagate, rbname ## _copy, rbname ## _rotate \
};
#define RB_RED 0
#define RB_BLACK 1
#define __rb_parent(pc) ((struct rb_node *)(pc & ~3))
#define __rb_color(pc) ((pc) & 1)
#define __rb_is_black(pc) __rb_color(pc)
#define __rb_is_red(pc) (!__rb_color(pc))
#define rb_color(rb) __rb_color((rb)->__rb_parent_color)
#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color)
#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color)
static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
{
rb->__rb_parent_color = rb_color(rb) | (unsigned long)p;
}
static inline void rb_set_parent_color(struct rb_node *rb,
struct rb_node *p, int color)
{
rb->__rb_parent_color = (unsigned long)p | color;
}
static inline void
__rb_change_child(struct rb_node *old, struct rb_node *new,
struct rb_node *parent, struct rb_root *root)
{
if (parent) {
if (parent->rb_left == old)
parent->rb_left = new;
else
parent->rb_right = new;
} else
root->rb_node = new;
}
extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root,
void (*augment_rotate)(struct rb_node *old, struct rb_node *new));
static __always_inline struct rb_node *
__rb_erase_augmented(struct rb_node *node, struct rb_root *root,
const struct rb_augment_callbacks *augment)
{
struct rb_node *child = node->rb_right, *tmp = node->rb_left;
struct rb_node *parent, *rebalance;
unsigned long pc;
if (!tmp) {
/*
* Case 1: node to erase has no more than 1 child (easy!)
*
* Note that if there is one child it must be red due to 5)
* and node must be black due to 4). We adjust colors locally
* so as to bypass __rb_erase_color() later on.
*/
pc = node->__rb_parent_color;
parent = __rb_parent(pc);
__rb_change_child(node, child, parent, root);
if (child) {
child->__rb_parent_color = pc;
rebalance = NULL;
} else
rebalance = __rb_is_black(pc) ? parent : NULL;
tmp = parent;
} else if (!child) {
/* Still case 1, but this time the child is node->rb_left */
tmp->__rb_parent_color = pc = node->__rb_parent_color;
parent = __rb_parent(pc);
__rb_change_child(node, tmp, parent, root);
rebalance = NULL;
tmp = parent;
} else {
struct rb_node *successor = child, *child2;
tmp = child->rb_left;
if (!tmp) {
/*
* Case 2: node's successor is its right child
*
* (n) (s)
* / \ / \
* (x) (s) -> (x) (c)
* \
* (c)
*/
parent = successor;
child2 = successor->rb_right;
augment->copy(node, successor);
} else {
/*
* Case 3: node's successor is leftmost under
* node's right child subtree
*
* (n) (s)
* / \ / \
* (x) (y) -> (x) (y)
* / /
* (p) (p)
* / /
* (s) (c)
* \
* (c)
*/
do {
parent = successor;
successor = tmp;
tmp = tmp->rb_left;
} while (tmp);
parent->rb_left = child2 = successor->rb_right;
successor->rb_right = child;
rb_set_parent(child, successor);
augment->copy(node, successor);
augment->propagate(parent, successor);
}
successor->rb_left = tmp = node->rb_left;
rb_set_parent(tmp, successor);
pc = node->__rb_parent_color;
tmp = __rb_parent(pc);
__rb_change_child(node, successor, tmp, root);
if (child2) {
successor->__rb_parent_color = pc;
rb_set_parent_color(child2, parent, RB_BLACK);
rebalance = NULL;
} else {
unsigned long pc2 = successor->__rb_parent_color;
successor->__rb_parent_color = pc;
rebalance = __rb_is_black(pc2) ? parent : NULL;
}
tmp = successor;
}
augment->propagate(tmp, NULL);
return rebalance;
}
static __always_inline void
rb_erase_augmented(struct rb_node *node, struct rb_root *root,
const struct rb_augment_callbacks *augment)
{
struct rb_node *rebalance = __rb_erase_augmented(node, root, augment);
if (rebalance)
__rb_erase_color(rebalance, root, augment->rotate);
}
#endif /* _LINUX_RBTREE_AUGMENTED_H */

106
src/entry/sfh_internal.h Normal file
View File

@@ -0,0 +1,106 @@
#include<zt_hash.h>
#include<interval_index.h>
#include<mesa_fuzzy.h>
#ifndef __SFH_INTERNAL_H_INCLUDE_
#define __SFH_INTERNAL_H_INCLUDE_
#define ROLLING_WINDOW 7
#define BLOCKSIZE_MIN 3
#define HASH_PRIME 0x01000193
#define HASH_INIT 0x28021967
#define CALCULATE 0
#define MODIFY 1
#define EXPECT_SIGNATURE_LEN 64
#define MEMORY_OCCUPY 3
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#ifndef container_of
#define container_of(ptr, type, member) ({ \
const typeof( ((type *)0)->member ) *__mptr = (ptr); \
(type *)( (char *)__mptr - offsetof(type,member) );})
#endif
#define DEBUG (0)
int hash_length;
//int count = 0;
struct roll_state_t
{
unsigned char window[ROLLING_WINDOW];
unsigned char pad[1];
unsigned int h1, h2, h3;
unsigned int n;
};
typedef struct
{
char mbuf[ROLLING_WINDOW-1];
char pad[8-ROLLING_WINDOW+1];
int slice_num;
unsigned int msize;
struct zt_state_t ps; //partial strong hash value
struct zt_state_t s_state;//strong hash state
unsigned long long left_offset;
unsigned long long right_offset;
struct roll_state_t r_state;
unsigned int * r_array; //array to store rolling hash value
unsigned int r_cnt;
unsigned int r_size;
struct zt_state_t * s_array; //array to store strong(Tillichi-Zemor) hash value
unsigned int s_cnt; //always point to the next available position
unsigned int s_size;
}sfh_seg_t;
typedef struct
{
unsigned long long orilen;
IVI_t * ivi; //ÿһ<C3BF><D2BB>handle<6C><65><EFBFBD>汣<EFBFBD><E6B1A3>һ<EFBFBD><D2BB>IVIָ<49>룬һ<EBA3AC><D2BB>IVI<56><49><EFBFBD>汣<EFBFBD><E6B1A3><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>Ƭ
unsigned long long effective_length;
unsigned long long blocksize;
unsigned long long fuzzy_node_memory;
unsigned long long IVI_memory;
unsigned long long length_increase;
int s_state_cnt;
unsigned int sim_tuned_rs_cnt;//rolling state count after a tune simulation
int do_tune;
}fuzzy_handle_inner_t;
typedef struct
{
char * data; //<2F><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>char<61><72><EFBFBD><EFBFBD>
unsigned int size;
unsigned int offset; //<2F><><EFBFBD>鳤<EFBFBD><E9B3A4>
unsigned long long first_ZTH_offset;
unsigned long long last_ZTH_offset;
char last_char;
}final_result;
typedef struct
{
unsigned long long first_ZTH_offset;
unsigned long long last_ZTH_offset;
unsigned long long hash_length;
}final_length;
sfh_seg_t* create_sfh_seg(fuzzy_handle_inner_t * _handle);
int destroy_sfh_seg(sfh_seg_t*p);
unsigned long long get_blocksize(unsigned long long orilen);
int sfh_merge_seg(fuzzy_handle_inner_t * _handle,sfh_seg_t * seg, sfh_seg_t * next_seg, unsigned long long blocksize);
int sfh_update_seg(fuzzy_handle_inner_t * _handle,sfh_seg_t * p, const char * data, unsigned long data_size, unsigned long long blocksize);
unsigned int segment_overlap(fuzzy_handle_inner_t * handle, unsigned int size, unsigned long long offset, const char * data);
void sfh_tune_seg(IVI_seg_t * seg, void * user_para);
void sfh_output_state(IVI_seg_t * seg, void * user_para);
void fuzzy_hash_length(IVI_seg_t * seg, void * user_para);
unsigned long long fuzzy_status(fuzzy_handle_t * handle, int type);
#endif

234
src/entry/zt_hash.h Normal file
View File

File diff suppressed because one or more lines are too long

71
src/inc_internal/field_stat.h Normal file
View File

@@ -0,0 +1,71 @@
#ifndef H_SCREEN_STAT_H_INCLUDE
#define H_SCREEN_STAT_H_INCLUDE
#include <stdio.h>
#ifndef __cplusplus
#error("This file should be compiled with C++ compiler")
#endif
enum field_dsp_style_t
{
FS_STYLE_FIELD=0,
FS_STYLE_COLUMN,
FS_STYLE_LINE,
FS_STYLE_STATUS
};
enum field_calc_algo
{
FS_CALC_CURRENT=0,
FS_CALC_SPEED
};
enum field_op
{
FS_OP_ADD=1,
FS_OP_SET
};
typedef void* screen_stat_handle_t;
enum FS_option
{
OUTPUT_DEVICE, //VALUE is a const char*, indicate a file path string, SIZE = strlen(string+'\0')+1.DEFAULT:output to stdout.
PRINT_MODE, //VALUE is an interger,1:Rewrite ,2: Append. SIZE=4,DEFALUT:REWRITE.
STAT_CYCLE, //VALUE is an interger idicate interval seconds of every output, SIZE=4 ,DEFUALT:2 seconds.
PRINT_TRIGGER, //VALUE is an interger,1:Do print,0: Don't print.SIZE=4.DEFAULT:1.
CREATE_THREAD,//VALUE is an interger,1: Create a print thread,0:not create,output by call passive_output function,
//and the STAT_CYCLE is meaningless.SIZE=4,DEFAULT:0.
ID_INVISBLE,//value is field_id/status_id/column_id, not output this string, SIZE=4,DEFAULT: shutdown NO one.
};
//Always success.
screen_stat_handle_t FS_create_handle(void);
int FS_set_para(screen_stat_handle_t handle, enum FS_option type,const void* value,int size);
void FS_start(screen_stat_handle_t handle);
void FS_stop(screen_stat_handle_t* handle);
//return field_id/line_id/column_id greater than zero if success,return an interger less than zero if failed.
int FS_register(screen_stat_handle_t handle,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name);
//numerator_id and denominator_id must be column/field/status style.
//scaling: negative value: zoom in; positive value: zoom out;
int FS_register_ratio(screen_stat_handle_t handle,int numerator_id,int denominator_id,int scaling,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name);
//id: when id's type is FIELD , column_id is ignore.
int FS_operate(screen_stat_handle_t handle,int id,int column_id,enum field_op op,long long value);
void FS_passive_output(screen_stat_handle_t handle);
screen_stat_handle_t init_screen_stat(FILE* output_fp,int stat_cycle,int screen_print_trigger);
//return field_id >=0 when success, return -1 when failed.
int stat_field_register(screen_stat_handle_t handle,const char* field_name);
//return >=0 when success, return -1 when failed.
#define FS_OP_TYPE_ADD (FS_OP_ADD)
#define FS_OP_TYPE_SET (FS_OP_SET)
int stat_field_operation(screen_stat_handle_t handle,int field_id,int operation,long long value);
#endif

24
src/inc_internal/rulescan.h
View File

@@ -23,10 +23,21 @@ extern "C"
{
#endif
/* <20><><EFBFBD>岻ͬ<E5B2BB><CDAC>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
enum ScanType
{
SCANTYPE_DEFAULT = 0,
SCANTYPE_DETAIL_RESULT = 1,
SCANTYPE_REGEX_GROUP = 2
};
#define MAX_REGEX_GROUP_NUM 5 /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD>֧<EFBFBD>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĸ<EFBFBD><C4B8><EFBFBD> */
#define MAX_EXPR_ITEM_NUM (1U<<3) /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>MAX_EXPR_ITEM_NUM<55><4D><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
#define MAX_MATCH_POS_NUM 1024 /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õĸ<C3B5><C4B8><EFBFBD> */
#define MATCH_POS_NUM_INC 64 /* ÿ<><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ص<EFBFBD><D8B5><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õĸ<C3B5><C4B8><EFBFBD><EFBFBD><EFBFBD>ʼֵ<CABC><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵ */
/* <20><><EFBFBD>岻ͬ<E5B2BB>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
/* <20><><EFBFBD>岻ͬ<E5B2BB>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
const unsigned int RULETYPE_STR = 0; /* <20>ַ<EFBFBD><D6B7><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ƹ<EFBFBD><C6B9><EFBFBD> */
const unsigned int RULETYPE_REG = 1; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD> */
const unsigned int RULETYPE_INT = 2; /* <20><>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
@@ -195,6 +206,17 @@ extern "C"
*/
void * rulescan_initialize(unsigned int max_thread_num);
/*
<09><><EFBFBD>ܣ<EFBFBD>
<09><><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>rulescan_update֮ǰ<D6AE>ɶ<EFBFBD><C9B6>ε<EFBFBD><CEB5>ã<EFBFBD>ÿ<EFBFBD><C3BF><EFBFBD><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
instance[in]: ɨ<><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ָ<EFBFBD>룻
scan_type_flag[in]: ɨ<><C9A8><EFBFBD><EFBFBD><EFBFBD>͵ı<CDB5>־<EFBFBD><D6BE>Ϣ<EFBFBD><CFA2>0<EFBFBD><30><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õ<EFBFBD><C3B5><EFBFBD>Ϣ<EFBFBD><CFA2>1<EFBFBD><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD>õ<EFBFBD><C3B5><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD>Dz<EFBFBD><C7B2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2>2<EFBFBD><32><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>λ<EFBFBD><CEBB><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ϣ
<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
1<><31><EFBFBD><EFBFBD>ȷ<EFBFBD><C8B7><EFBFBD>ã<EFBFBD>-1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʧ<EFBFBD>ܡ<EFBFBD>
*/
int rulescan_set_param(void * instance, int scan_type_flag);
/*
<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD>̬ע<CCAC><D7A2>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>󡣶<EFBFBD><F3A1A3B6><EFBFBD>ͬһ<CDAC><D2BB>instance<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬʱ<CDAC>ж<EFBFBD><D0B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>̡߳<DFB3>
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

2
test/Makefile
View File

@@ -2,6 +2,6 @@ LIBS=../lib/libmaatframe.so
INC=-I../inc/ -I/usr/include/MESA/
all:
g++ -o maat_test -g -Wall maat_test.cpp $(INC) $(LIBS)
g++ -o digest_gen -g digest_gen.c -I../src/inc_internal/ ../lib/libmaatframe.so
g++ -o digest_gen -g digest_gen.c $(INC) $(LIBS)
clean:
rm maat_test test.log* -f

111
test/maat_json.json
View File

@@ -181,13 +181,122 @@
"table_type": "digest",
"table_content": {
"raw_len": 1160164,
"digest": "FsOmaK3utZafWYt/i[7203:46299992]",
"digest": "12288:UChtbFS6pypdTy4m2[0:1160163]",
"cfds_level": 3
}
}
]
}
]
},
{
"compile_id": 128,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_8",
"regions": [
{
"table_name": "HTTP_REGION",
"table_type": "expr_plus",
"table_content": {
"district": "URL",
"keywords": "abckkk&123",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 129,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "utf8_中文",
"is_valid": "yes",
"groups": [
{
"group_name": "group_9",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "C#中国",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 130,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "utf8_维语",
"is_valid": "yes",
"groups": [
{
"group_name": "group_10",
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "string",
"table_content": {
"keywords": "2010&يىلىدىكى",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 131,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"effective_rage": 0,
"user_region": "utf8_维语2",
"is_valid": "yes",
"groups": [
{
"group_name": "group_11",
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "string",
"table_content": {
"keywords": "سىياسىي",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [

459
test/maat_test.cpp
View File

@@ -13,7 +13,11 @@
#include <MESA/stream.h>
#include <sys/types.h>//fstat
#include <sys/stat.h>//fstat
#include <unistd.h>
#include <dirent.h>
extern int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *));
void Maat_read_entry_start_cb(int update_type,void* u_para)
{
return;
@@ -36,113 +40,101 @@ void Maat_read_entry_finish_cb(void* u_para)
{
return;
}
int main()
void print_maat_ret(int ret)
{
Maat_feather_t feather=NULL;
int cb_table_id=-1,url_scan_table_id=-1,size_scan_table_id=-1,ip_scan_table_id=-1,digest_scan_table_id=-1;
int ret=-1;
int g_iThreadNum=4;
const char* table_info_path="./table_info.conf";
const char* json_path="./maat_json.json";
// const char* ful_cfg_dir="./maat_json.json_iris_tmp/index";
// const char* inc_cfg_dir="./rules/inc/index";
const char* log_file="./test.log";
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char* digest_test_file="./digest_test.data";
struct stat digest_fstat;
unsigned long long read_size=0,scan_offset=0;
char digest_test_buff[4096]={0};
int scan_val=2015;
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
return;
}
const char* print_maat_rule(struct Maat_rule_t* result,int ret)
{
static char buff[1024]={0};
int i=0,j=0;
switch(ret)
{
case -1:
snprintf(buff,sizeof(buff),"ret=%d,scan error.",ret);
break;
case -2:
snprintf(buff,sizeof(buff),"ret=%d,hit current region,but not hit compile rule.",ret);
break;
case 0:
snprintf(buff,sizeof(buff),"ret=0,nothing hit.");
break;
default://>0
j=snprintf(buff,sizeof(buff),"hit %d rules, hit ruleid=",ret);
for(i=0;i<ret;i++)
{
j+=snprintf(buff+j,sizeof(buff)-j,"%d ",result[i].config_id);
}
break;
}
return buff;
}
int test_string_full_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
scan_status_t mid=NULL;
void *logger=MESA_create_runtime_log_handle(log_file,0);
feather=Maat_summon_feather_json(g_iThreadNum,
table_info_path,
json_path,
logger);
// feather=Maat_summon_feather(g_iThreadNum,table_info_path,ful_cfg_dir,inc_cfg_dir,logger);
if(feather==NULL)
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Maat initial error, see %s\n",log_file);
printf("Database table %s register failed.\n",table_name);
return -1;
}
cb_table_id=Maat_table_register(feather,"QD_ENTRY_INFO");
if(cb_table_id==-1)
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
print_maat_ret(ret);
return ret;
}
int test_intval_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
int scan_val=2015;
struct Maat_rule_t result[4];
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table QD_ENTRY_INFO register failed.");
}
else
{
ret=Maat_table_callback_register(feather, cb_table_id,
Maat_read_entry_start_cb,
Maat_read_entry_cb,
Maat_read_entry_finish_cb,
logger);
if(ret<0)
{
printf("Maat callback register table QD_ENTRY_INFO error.");
}
}
url_scan_table_id=Maat_table_register(feather,"HTTP_URL");
if(url_scan_table_id==-1)
{
printf("Database table HTTP_URL register failed.");
printf("Database table %s register failed.",table_name);
}
else
{
ret=Maat_full_scan_string(feather, url_scan_table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
&mid, 0);
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
ret=Maat_scan_intval(feather, table_id, scan_val, result,4,mid, 0);
print_maat_ret(ret);
}
Maat_clean_status(&mid);
size_scan_table_id=Maat_table_register(feather,"CONTENT_SIZE");
if(size_scan_table_id==-1)
return ret;
}
int test_str_stream_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table CONTENT_SIZE register failed.");
printf("Database table %s register failed.\n",table_name);
return -1;
}
else
{
ret=Maat_scan_intval(feather, size_scan_table_id, scan_val, result,4,&mid, 0);
switch(ret)
{
case -1:
printf("scan error.\n");
break;
case -2:
printf("hit current region,but not hit compile rule.\n");
break;
case 0:
printf("nothing hit\n");
break;
default://>0
printf("hit %d rules\n",ret);
break;
}
}
Maat_clean_status(&mid);
struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
stream_para_t sp=Maat_stream_scan_string_start(feather,url_scan_table_id,0);
stream_para_t sp=Maat_stream_scan_string_start(feather,table_id,0);
int detail_ret=0;
if(sp==NULL)
{
@@ -151,15 +143,21 @@ int main()
}
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com")
,result,4,hit_detail,10
,&detail_ret,&mid);
,&detail_ret,mid);
ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data)
,result,4,hit_detail,10
,&detail_ret,&mid);
,&detail_ret,mid);
Maat_stream_scan_string_end(&sp);
struct ipaddr ipv4_addr,ipv6_addr;
free(hit_detail);
print_maat_ret(ret);
return ret;
}
int test_ipv4_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
struct stream_tuple4_v6 v6_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr));
v4_addr.source=htons(50001);
@@ -167,30 +165,45 @@ int main()
v4_addr.dest=htons(80);
ipv4_addr.v4=&v4_addr;
ip_scan_table_id=Maat_table_register(feather,"IP_CONFIG");
if(ip_scan_table_id==-1)
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table IP_CONFIG register failed.");
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv4_addr,6,result,4, &mid,0);
ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,result,4, mid,0);
if(ret>0)
{
printf("ipv4 scan hit compile rule id %d.\n",result[0].config_id);
}
}
Maat_clean_status(&mid);
return ret;
}
int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
struct ipaddr ipv6_addr;
struct stream_tuple4_v6 v6_addr;
ipv6_addr.addrtype=ADDR_TYPE_IPV6;
inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr));
v6_addr.source=htons(50001);
inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr));
v6_addr.dest=htons(80);
ipv6_addr.v6=&v6_addr;
if(ip_scan_table_id>=0)
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
ret=Maat_scan_proto_addr(feather,ip_scan_table_id,&ipv6_addr,6,result,4, &mid,0);
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0);
if(ret==-2)
{
printf("ipv6 scan hit region.\n");
@@ -201,12 +214,22 @@ int main()
}
}
Maat_clean_status(&mid);
return ret;
}
int test_digest_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int table_id=0,ret=0;
const char* digest_test_file="./digest_test.data";
struct stat digest_fstat;
unsigned long long read_size=0,scan_offset=0;
char digest_test_buff[4096]={0};
digest_scan_table_id=Maat_table_register(feather, "FILE_DIGEST");
if(digest_scan_table_id<0)
struct Maat_rule_t result[4];
stream_para_t sp=NULL;
table_id=Maat_table_register(feather, table_name);
if(table_id<0)
{
printf("registe table FILE_DIGEST error.\n");
printf("registe table %s error.\n",table_name);
return 0;
}
ret=stat(digest_test_file,&digest_fstat);
@@ -218,11 +241,11 @@ int main()
FILE* fp=fopen(digest_test_file,"r");
if(fp!=NULL)
{
sp=Maat_stream_scan_digest_start(feather, digest_scan_table_id, digest_fstat.st_size, 0);
sp=Maat_stream_scan_digest_start(feather, table_id, digest_fstat.st_size, 0);
while(0==feof(fp))
{
read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,&mid);
ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,mid);
scan_offset+=read_size;
if(ret>0)
{
@@ -237,8 +260,218 @@ int main()
printf("fopen %s error.\n",digest_test_file);
}
Maat_stream_scan_string_end(&sp);
Maat_clean_status(&mid);
Maat_burn_feather(feather);
free(hit_detail);
return ret;
}
int test_plugin_table(Maat_feather_t feather,const char* table_name,void* logger)
{
int table_id=0,ret=0;
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
}
else
{
ret=Maat_table_callback_register(feather, table_id,
Maat_read_entry_start_cb,
Maat_read_entry_cb,
Maat_read_entry_finish_cb,
logger);
if(ret<0)
{
printf("Maat callback register table %s error.\n",table_name);
}
}
return ret;
}
int test_url_encode(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
const char* url_utf8="www.google.com/?q=C%23%E4%B8%AD%E5%9B%BD";
const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA";
int table_id=0,ret=0;
struct Maat_rule_t result[4];
int found_pos[4];
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.",table_name);
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, url_utf8, strlen(url_utf8),
result,found_pos, 4,
mid, 0);
printf("URL encode scan utf8 url %s\n",print_maat_rule(result,ret));
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312),
result,found_pos, 4,
mid, 0);
printf("URL encode scan gb2312 url %s\n",print_maat_rule(result,ret));
return 0;
}
int test_unicode_esc(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
const char* test_data_dir="./testdata_uni2ascii";
struct dirent **namelist;
FILE* fp=NULL;
char file_path[256]={0};
char buff[4096];
size_t read_len=0;
int table_id=0,ret=0;
struct Maat_rule_t result[4];
stream_para_t sp=NULL;
int found_pos[4];
int n=0,i=0;
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed in function %s.\n",table_name,__FUNCTION__);
return -1;
}
n = my_scandir(test_data_dir, &namelist, NULL, (int (*)(const void*, const void*))alphasort);
if(n<0)
{
printf("%s open dir %s error.\n",__FUNCTION__,test_data_dir);
return -1;
}
for(i=0;i<n;i++)
{
if((strcmp(namelist[i]->d_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0))
{
continue;
}
snprintf(file_path,sizeof(file_path),"%s/%s",test_data_dir,namelist[i]->d_name);
fp=fopen(file_path,"rb");
if(fp==NULL)
{
printf("fopen %s error.\n",file_path);;
continue;
}
printf("%s processing %s\n",__FUNCTION__,file_path);
sp=Maat_stream_scan_string_start(feather,table_id,0);
if(sp==NULL)
{
printf("stream scan start failed.\n");
continue;
}
read_len=fread(buff,1,sizeof(buff),fp);
while(read_len>0)
{
ret=Maat_stream_scan_string(&sp,CHARSET_NONE,buff,read_len
,result,found_pos,4,mid);
read_len=fread(buff,1,sizeof(buff),fp);
if(ret>0)
{
printf("UNI2ASCII file %s,%s\n",file_path,print_maat_rule(result,ret));
}
}
Maat_stream_scan_string_end(&sp);
fclose(fp);
}
for(i=0;i<n;i++)
{
free(namelist[i]);
}
free(namelist);
return 0;
}
int test_expr_plus(Maat_feather_t feather,const char* table_name,scan_status_t* mid)
{
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* region_name="URL";
const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
table_id=Maat_table_register(feather,table_name);
if(table_id==-1)
{
printf("Database table %s register failed.\n",table_name);
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
if(ret>0)
{
printf("Should not hit without setting district.\n");
return -1;
}
ret=Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_DISTRICT,region_name,strlen(region_name)+1);
if(ret<0)
{
printf("set MAAT_SET_SCAN_DISTRICT failed.\n");
return -1;
}
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result,found_pos, 4,
mid, 0);
if(ret>0)
{
printf("Hit expr_plus rule %d.\n",result[0].config_id);
}
return ret;
}
int main(int argc,char* argv[])
{
Maat_feather_t feather=NULL;
int g_iThreadNum=4;
const char* table_info_path="./table_info.conf";
const char* json_path="./maat_json.json";
// const char* ful_cfg_dir="./maat_json.json_iris_tmp/index";
// const char* inc_cfg_dir="./rules/inc/index";
const char* log_file="./test.log";
const char* stat_file="./scan_staus.log";
int scan_detail=0;
scan_status_t mid=NULL;
void *logger=MESA_create_runtime_log_handle(log_file,0);
feather=Maat_feather(g_iThreadNum, table_info_path, logger);
Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0);
Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0);
Maat_set_feather_opt(feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
Maat_initiate_feather(feather);
if(feather==NULL)
{
printf("Maat initial error, see %s\n",log_file);
return -1;
}
test_plugin_table(feather, "QD_ENTRY_INFO",logger);
test_string_full_scan(feather, "HTTP_URL", &mid);
//not clean status here, to test_ipv4_scan make hit compile rule.
test_ipv4_scan(feather, "IP_CONFIG", &mid);
Maat_clean_status(&mid);
test_intval_scan(feather,"CONTENT_SIZE" , &mid);
Maat_clean_status(&mid);
test_ipv6_scan(feather, "IP_CONFIG", &mid);
Maat_clean_status(&mid);
test_digest_scan(feather,"FILE_DIGEST", &mid);
Maat_clean_status(&mid);
test_expr_plus(feather, "HTTP_REGION", &mid);
Maat_clean_status(&mid);
test_url_encode(feather, "HTTP_URL", &mid);
Maat_clean_status(&mid);
test_unicode_esc(feather,"KEYWORDS_TABLE",&mid);
Maat_clean_status(&mid);
sleep(4);
Maat_burn_feather(feather);
return 0;
}

7
test/table_info.conf
View File

@@ -1,16 +1,17 @@
#each collumn seperate with '\t'
#id (0~65535)
#name string
#type one of ip,expr,digest,intval,compile or plugin
#type one of ip,expr,expr_plus,digest,intval,compile or plugin
#src_charset one of GBK,BIG5,UNICODE,UTF8
#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
#do_merege yes or no
#id name type src_charset dst_charset do_merge
0 COMPILE compile GBK GBK no 0
1 GROUP group GBK GBK no 0
2 HTTP_URL expr GBK GBK/BIG5/UNICODE/UTF8 yes 128
3 KEYWORDS_TABLE expr GBK GBK/BIG5/UNICODE/UTF8 yes 0
2 HTTP_URL expr UTF8 GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8 yes 128
3 KEYWORDS_TABLE expr UTF8 GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex yes 0
4 IP_CONFIG ip GBK GBK no 0
5 CONTENT_SIZE intval GBK GBK no 0
6 QD_ENTRY_INFO plugin GBK GBK no 0
7 FILE_DIGEST digest GBK GBK no 0
8 HTTP_REGION expr_plus GBK GBK no 0

166
test/testdata_uni2ascii/original_Uygur_webpage.html Normal file
View File

@@ -0,0 +1,166 @@
<!DOCTYPE html>
<!-- saved from url=(0062)http://uy.ts.cn/homepage/content/2016-03/31/content_514152.htm -->
<html xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><img src="./orignial_webpage_files/dmpcm"><img src="./orignial_webpage_files/ping.gif"><img src="./orignial_webpage_files/pixel"><script src="./orignial_webpage_files/cm.js" charset="utf-8"></script><script src="./orignial_webpage_files/id.php" charset="utf-8"></script>
<title>مەركىزىي خىزمەتلەرنى چۆرىدەپ ئەقىل كۆرسىتىش، ۋەزىپىسىنى بەجانىدىل ئادا قىلىپ دادىل زىممىگە ئېلىش- تەڭرىتاغ تورى</title>
<link href="./orignial_webpage_files/H-ui.reset.css" rel="stylesheet">
<link href="./orignial_webpage_files/font-style.css" rel="stylesheet">
<link href="./orignial_webpage_files/style.css" rel="stylesheet">
<script src="./orignial_webpage_files/jquery-1.11.3.js"></script>
<script src="./orignial_webpage_files/jquery.SuperSlide.2.1.1.js"></script>
<script src="./orignial_webpage_files/js.js"></script>
<meta name="filetype" content="0">
<meta name="publishedtype" content="1">
<meta name="pagetype" content="1">
<meta name="contentid" content="514152">
<meta name="publishdate" content="2016-03-31">
<meta name="author" content="بۇئايشەم">
</head>
<body><link href="./orignial_webpage_files/jiathis_share.css" rel="stylesheet" type="text/css"><iframe frameborder="0" style="position: absolute; display: none; opacity: 0;"></iframe><div class="jiathis_style" style="position: absolute; z-index: 1000000000; display: none; top: 50%; left: 50%; overflow: auto;"></div><div class="jiathis_style" style="position: absolute; z-index: 1000000000; display: none; overflow: auto;"></div><iframe frameborder="0" src="./orignial_webpage_files/jiathis_utility.html" style="display: none;"></iframe>
<!-------------顶部-------------->
<div class="top" id="Top">
<iframe src="./orignial_webpage_files/node_15563.html" scrolling="no" allowtransparency="true" frameborder="0" width="100%" height="30"></iframe>
</div>
<div class="ch10"></div>
<!------导航-------->
<div class="layout" id="Benner">
<dl>
<dd>
<a href="http://uy.ts.cn/">
<img src="./orignial_webpage_files/logo.png">
</a>
</dd>
</dl>
<div>
<a target="_blank" href="http://bbs.xjtsnews.com/portal.php">مۇنبەر</a>
<a target="_blank" href="http://www.uzunyol.cn/app/">خەۋەر تېرمىنالى</a>
<a target="_blank" href="http://uy.ts.cn/zhuanti/index.htm">مەخسۇس سەھىپە</a>
<a target="_blank" href="http://dihan.com.cn/">دېھقانلار تورى</a>
<a target="_blank" href="http://gizit.ts.cn/" style="font-size: 12px;">يانفون گېزىتى</a>
<a target="_blank" href="http://www.uzunyol.cn/undidar/">ئۈندىدار سالونى</a>
</div>
</div>
<div class="ch"></div>
<!-------图片广告-------->
<!--广告开始，好几个轮转显示-->
<div class="layout" id="Top-adv1">
<div id="top-adv" class="top-adv">
<div class="hd">
<ul></ul>
</div>
<div class="bd">
<div class="tempWrap" style="overflow:hidden; position:relative; height:60px"><ul style="top: -170.961px; position: relative; padding: 0px; margin: 0px;">
<li style="height: 60px;"><a href="http://uy.ts.cn/2016lianghui/node_15669.htm" target="_blank"><img src="./orignial_webpage_files/0a0027000000183fb99008.jpg" border="0"></a></li><li style="height: 60px;"><a href="http://uy.ts.cn/13dong/node_15075.htm" target="_blank"><img src="./orignial_webpage_files/fc4dd4f7ea9e180834d202.jpg" border="0"></a></li><li style="height: 60px;"><a href="http://uy.ts.cn/hlwdh/node_15228.htm" target="_blank"><img src="./orignial_webpage_files/fc4dd4f7ea9e17db87af05.jpg" border="0"></a></li><li style="height: 60px;"><a href="http://uy.ts.cn/zhuanti/node_14902.htm" target="_blank"><img src="./orignial_webpage_files/fc4dd4f7ea9e17a575c701.jpg" border="0"></a></li><li style="height: 60px;"><a href="http://uy.ts.cn/topic/60zhounian/node_14703.htm" target="_blank"><img src="./orignial_webpage_files/3c46d84ef0261776eeb201.jpg" border="0"></a></li>
</ul></div>
</div>
</div>
<div class="clearfix"></div>
</div>
<!--广告结束-->
<div class="ch20"></div>
<!---------内容---------->
<div class="layout" id="View">
<div id="x_box"></div>
<div id="b_box" style="width: 200px;">
<a href="http://uy.ts.cn/node_881.htm" target="_blank" class="">باش خەۋەر</a>
</div>
<h2 id="Title">
مەركىزىي خىزمەتلەرنى چۆرىدەپ ئەقىل كۆرسىتىش، ۋەزىپىسىنى بەجانىدىل ئادا قىلىپ دادىل زىممىگە ئېلىش
</h2>
<span id="User">بۇئايشەم</span><span id="Time">2016-03-31 11:17:39</span><span id="Source"><a href="" target="_blank">شىنجاڭ گېزىتى</a></span>
<div class="ch"></div>
<div id="Content">
<!--enpproperty <articleid>514152</articleid><date>2016-03-31 11:17:39.0</date><author></author><title>مەركىزىي خىزمەتلەرنى چۆرىدەپ ئەقىل كۆرسىتىش، ۋەزىپىسىنى بەجانىدىل ئادا قىلىپ دادىل زىممىگە ئېلىش</title><keyword></keyword><subtitle></subtitle><introtitle></introtitle><siteid>82</siteid><nodeid>881</nodeid><nodename>باش خەۋەر</nodename><nodesearchname></nodesearchname>/enpproperty--><!--enpcontent--><!--enpcontent--><p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: center" align="center"><font face="ALKATIP Kitab Tom">&nbsp;<strong>ئاپتونوم رايوننىڭ 2010–يىلىدىكى ئاپتونوم رايونلۇق پارتكوم سىياسىي كېڭەش خىزمىتى يىغىنىدىن بۇيانقى سىياسىي كېڭەش خىزمىتى توغرىسىدا ئومۇمىي بايان </strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: center" align="center"><font face="ALKATIP Kitab Tom"><strong>ياۋ تۇڭ</strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;‹‹قەلبلەرنى مۇجەسسەملەش، ئەقىل كۆرسىتىش، كۈچ قوشۇش›› ج ك پ مەركىزىي كومىتېتى سىياسىي بىيۇروسىنىڭ ئەزاسى، ئاپتونوم رايونلۇق پارتكومنىڭ شۇجىسى جاڭ چۈنشيەننىڭ سىياسىي كېڭەشكە قويغان بۇ تەلىپى ئاپتونوم رايوننىڭ سىياسىي كېڭەش خىزمىتىنىڭ يۆنىلىشىنى كۆرسىتىپ بېرىپ، ئۇنى پىرىنسىپ بىلەن تەمىنلەپ، ئەزالارنىڭ يېڭى ۋەزىيەتتە سىياسىي كېڭەش خىزمىتىنى ياخشى ئىشلەش ئىشەنچى ۋە ئىرادىسىنى كۈچەيتتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق پارتكوم سىياسىي كېڭەش خىزمىتىگە ئىزچىل يۈكسەك ئەھمىيەت بېرىپ كەلدى، شۇجى جاڭ چۈنشيەن سىياسىي كېڭەش خىزمىتىگە ناھايىتى كۆڭۈل بۆلدى ۋە ئۇنى قوللىدى، 2010 – يىلى ئاپتونوم رايونلۇق پارتكومنىڭ سىياسىي كېڭەش خىزمىتى يىغىنى ئېچىلغاندىن بۇيان، شۇجى جاڭ چۈنشيەن سىياسىي كېڭەش خىزمىتىنى كۈچەيتىش توغرىسىدا كۆپ قېتىم تەستىقىي يوليۇرۇق بەردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;نەچچە يىلدىن بۇيان ئاپتونوم رايونلۇق سىياسىي كېڭەش پارتىيە رەھبەرلىكىدە چىڭ تۇرۇشنى باشتىن – ئاخىر ۋەزىپىسىنى ئادا قىلىشنىڭ بىرىنچى تەلىپى قىلىپ، سىياسىي كېڭەشنىڭ ۋەتەنپەرۋەرلىك بىرلىك سەپ تەشكىلاتىلىق مۇھىم رولىنى تولۇق جارى قىلدۇرۇپ، ئورتاق تونۇشنى كەڭ دائىرىدە مۇجەسسەملىدى؛ دىققەتنى ئىجتىمائىي مۇقىملىق ۋە ئەبەدىي ئەمىنلىككە مەركەزلەشتۈرۈپ، دائىمىي كومىتېت يىغىنى ئېچىپ، بىر قاتار مۇھىم مەسىلىلەر توغرىسىدا تەدبىر، تەكلىپلەرنى ئوتتۇرىغا قويدى؛ ئايلىق كېڭىشىش سۆھبەت يىغىنى ئېچىشنى ئىجادىي يولغا قويۇپ، كېڭىشىپ بىرلىككە كەلگەن پىكىرلەرنى ئاپتونوم رايونلۇق پارتكوم، خەلق ھۆكۈمىتىنىڭ پايدىلىنىشىغا سۇندى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەشنىڭ سىياسىي كېڭەش شىنجاڭ ئۇيغۇر ئاپتونوم رايونلۇق 11 – نۆۋەتلىك كومىتېتى 1 – يىغىنىدىن بۇيانقى ۋەزىپىسىنى ئادا قىلىش خاتىرىسىگە نەزەر سالغاندا، كۆپلىگەن تەكلىپ، تەۋسىيەلەر، بىر قاتار تەكشۈرۈپ تەتقىق قىلىش دوكلاتلىرىنىڭ ھەممىسىدە شىنجاڭدىكى ھەر دەرىجىلىك سىياسىي كېڭەش تەشكىلاتلىرى ۋە سىياسىي كېڭەش ئەزالىرىنىڭ شىنجاڭنىڭ ئىقتىسادىي، ئىجتىمائىي تەرەققىياتى ئۈچۈن قەلبلەرنى مۇجەسسەملەپ، ئەقىل كۆرسىتىپ، كۈچ چىقارغانلىقىدىن ئىبارەت جانلىق ئەمەلىيەت خاتىرىلەنگەن بولۇپ، بۇلار شىنجاڭدىكى ھەر دەرىجىلىك سىياسىي كېڭەش تەشكىلاتلىرى ۋە سىياسىي كېڭەش ئەزالىرىنىڭ دىققەتنى ئاپتونوم رايوننىڭ مەركىزىي خىزمىتىگە مەركەزلەشتۈرۈپ فۇنكسىيەسىنى ئادا قىلىپ، سىياسىي كېڭەشنىڭ كېڭىشىش ئاساسىدىكى دېموكراتىيە قەدىمىنى ئىلگىرى سۈرۈپ، ئىستىل ۋە ئىقتىدار قۇرۇلۇشىنى كۈچەيتىپ، ئەمەلىي ئۈنۈمنى ئاشۇرۇش داۋامىدىكى مەزمۇت قەدەملىرىنىڭ شاھىتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom"><strong>&nbsp;&nbsp;&nbsp;&nbsp;كۈچنى مەركەزلەشتۈرۈپ، ئىتتىپاقلىقنى ئىلگىرى سۈرۈپ، قەلبلەرنى مۇجەسسەملەپ، بىرلىكتە گۈزەل شىنجاڭ قۇرۇپ چىقىش</strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەشنىڭ ئەزاسى ۋېي تيەنجې كېسەللىك سەۋەبىدىن نامراتلىشىپ كەتكەن بىر قازاق ئائىلىسىنىڭ قىيىنچىلىقتىن قۇتۇلۇشىغا 20 يىل ياردەم بەردى، ئەڭ دەسلەپتە مەبلەغ چىقىرىپ يېڭى ئۆي سېلىپ بېرىش، قوي ئېلىپ بېرىشتىن، ئەر خوجايىننىڭ قومۇل شەھىرىدە مۇقىم خىزمەت تېپىپ، ئولتۇراقلىشىشى ۋە ئۇنىڭ ئىككى بالىسىنىڭ ئوتتۇرا مەكتەپنى، ئالىي مەكتەپنى تاماملاپ، خىزمەتكە چىقىشىغىچە ئۇلارغا ئىزچىل ئىقتىسادىي جەھەتتىن ياردەم قىلدى. چىڭگىل ناھىيەلىك سىياسىي كېڭەشنىڭ ئەزاسى ئامىنە ساۋۇت 50 ياشتىن ئاشقان، ئائىلە ئەھۋالى ئانچە ياخشى بولمىغان يەككە سودا – سانائەتچى، ئۇ ئىقتىسادچانلىق بىلەن تۇرمۇش كەچۈرۈپ، كۆپ يىللاردىن بۇيان يىغقان 100 مىڭ يۈەندىن كۆپرەك پۇلىغا كىچىك چىڭگىل دەرياسىغا پولات ئارغامچىلىق ئاسما كۆۋرۈك سالدۇرۇپ، ئىككى قىرغاقتىكى ئۈچ كەنتتىكى دېھقان – چارۋىچىلارنىڭ يول يۈرۈشىگە زور قولايلىق يارىتىپ بەردى...</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;يېقىنقى يىللاردىن بۇيان ھەر مىللەت، ھەر ساھە سىياسىي كېڭەش ئەزالىرى مىللەتلەر ئىتتىپاقلىقىنى باشلامچىلىق بىلەن قوغداپ، مىللەتلەر ئىتتىپاقلىقىغا بۇزغۇنچىلىق قىلىدىغان بارلىق سۆز، ھەرىكەتلەرگە بايرىقى روشەن ھالدا قارشى تۇرۇپ، ئەمەلىي ھەرىكىتى بىلەن مىللەتلەرنىڭ ئىناق ئۆتۈشى، بىر ياقىدىن باش چىقىرىشى، ئىناق راۋاجلىنىشىغا تۈرتكە بولۇپ، پارتىيە – گۇرۇھلار، تەشكىلاتلار، ھەر مىللەت، ھەر قاتلامدىكىلەرنىڭ بۈيۈك ئىتتىپاقلىقى، بۈيۈك بىرلىشىشىنى ئىلگىرى سۈردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;2016 – يىلى 1 – ئايدا ئاپتونوم رايونلۇق سىياسىي كېڭەش بەنگۇڭتىڭى رەھبەرلىرى سىياسىي كېڭەش ئورگىنىدىن مارالبېشى ناھىيەسىنىڭ دۆلەتباغ يېزىسى قۇرۇقتېرەك كەنتىدە تۇرۇشلۇق خىزمەت گۇرۇپپىسىدىكىلەر توغرىسىدا يېزىلغان بىرلەشمە ئىمزالىق خەتنى تاپشۇرۇۋالدى، خەتنى مەزكۇر كەنتتىكى ‹‹تۆت پېشقەدەم››لەر، دىنىي زاتلار ۋە ئامما ۋەكىلىدىن بولۇپ 17 كىشى بىرلىشىپ يازغان بولۇپ، ئۇلار خەتتە پۈتۈن كەنتتىكىلەرگە ۋاكالىتەن كۈچلۈك ئارزۇسىنى ئىپادىلەپ: ‹‹كەنتتە تۇرۇشلۇق كادىر ئېنگۇسۇم زەينىدىننىڭ كەنتىمىزدە يەنە داۋاملىق بىر يىل تۇرۇشىنى ئۈمىد قىلىمىز›› دەپ يازغانىدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئىككى يىلدىن بۇيان ئاپتونوم رايونلۇق پارتكومنىڭ بىر تۇتاش ئورۇنلاشتۇرۇشى بويىچە شىنجاڭدىكى ھەر دەرىجىلىك سىياسىي كېڭەش تەشكىلاتلىرىدىن 1900دىن كۆپرەك سىياسىي كېڭەش ئەزاسى ۋە 630دىن كۆپرەك سىياسىي كېڭەش كادىرى شىنجاڭدىكى 14 ۋىلايەت، ئوبلاست، شەھەردىكى 1066 كەنتكە چوڭقۇر چۆكۈپ، ئۈچ ئەل پائالىيىتىنى كەڭ قانات يايدۇرۇپ، ئامما بىلەن تاماقتا، ئەمگەكتە بىللە بولۇپ، ئەڭ زور دەرىجىدە ئەلرايىنى مۇجەسسەملەپ، ئاممىنىڭ كۆڭلىنى ئۇتۇپ، ھەر مىللەت ئاممىغا سىياسىي كېڭەشنىڭ ئۆزلىرىگە ناھايىتى يېقىن ئىكەنلىكىنى ھەقىقىي ھېس قىلدۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش يەنە پارتىيە – گۇرۇھلار، تەشكىلاتلار ھەمكارلىقىنى كۈچەيتىشنى مۇھىم خىزمەت قىلىپ، دېموكراتىك پارتىيە – گۇرۇھلار، سودا – سانائەتچىلەر بىرلەشمىسى، خەلق تەشكىلاتلىرى ۋە پارتىيە – گۇرۇھسىز زاتلارنىڭ دېموكراتىك ھوقۇقىغا تولۇق ھۆرمەت قىلىپ، تەتقىق قىلىش، كېڭىشىش ئاساسىدا پىلان تۈزۈش قاتارلىق مۇھىم ئىشلاردا ئۇلاردىن تەكلىپ – پىكىر ئېلىشقا ئەھمىيەت بېرىپ، ئۇلارنىڭ ۋەزىپىسىنى ئۈنۈملۈك ئادا قىلىشىغا پائال سەھنە ھازىرلاپ بېرىپ، ئاپتونوم رايوننىڭ ئىسلاھات، تەرەققىيات، مۇقىملىقىغا ئەقىل – پاراسىتى ۋە كۈچىنى بېغىشلىدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom"><strong>&nbsp;&nbsp;&nbsp;&nbsp;ۋەزىپىلەرنى دادىل زىممىگە ئېلىپ، ياخشى ئەقىل كۆرسىتىپ، ئىناقلىقنى ئىلگىرى سۈرۈپ، ئەبەدىي ئەمىنلىكنى قوغداش</strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;مەملىكەتلىك سىياسىي كېڭەشنىڭ شىنجاڭدىكى ئەزاسى، ئاپتونوم رايونلۇق سىياسىي كېڭەشنىڭ رەئىسى نۇرلان ئابىلمەجىن 12 – مارت سىياسىي كېڭەش مەملىكەتلىك 12 – نۆۋەتلىك كومىتېتىنىڭ 4 – يىغىنىدا ‹‹ئەسەبىيلىكنى تۈگىتىشنى چوڭقۇر ئىلگىرى سۈرۈپ، دۆلەتنىڭ بىخەتەرلىكىنى قەتئىي قوغدايلى›› دېگەن تېمىدا سۆز قىلىپ، دۆلەتنىڭ ئەڭ يۇقىرى كېڭىشىش سەھنىسىدە دىن ئەسەبىيلىكى ئىدىيەسىدىن ئىبارەت ‹‹يامان سۈپەتلىك ئۆسمە››نى يوقىتىپ، دۆلەتنىڭ بىخەتەرلىكىنى قوغداپ، ‹‹بىر بەلباغ، بىر يول›› ئىستراتېگىيەسىنى ئوڭۇشلۇق يولغا قويۇشقا كاپالەتلىك قىلىشنى ئوتتۇرىغا قويدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;‹‹شىنجاڭ ئاۋازى›› خەلق سارىيىدا ياڭرىدى، بۇ شىنجاڭدىكى 23 مىليون ھەر مىللەت ئاممىنىڭ ئورتاق قەلب ساداسى! يېقىنقى نەچچە يىلدىن بۇيان ئاپتونوم رايونلۇق سىياسىي كېڭەش ئىجتىمائىي مۇقىملىق ۋە ئەبەدىي ئەمىنلىكتىن ئىبارەت بۇ باش نىشاننى چۆرىدەپ، ‹‹ئۈچ خىل كۈچ››كە زەربە بېرىش، دىن ئەسەبىيلىكى ئىدىيەسىنىڭ سىڭىپ كىرىشىنى چەكلەش، دىن ئىناقلىقىغا تەسىر يەتكۈزىدىغان ئامىللارنى تۈگىتىش، تىنچ شىنجاڭ قۇرۇشنى ئىلگىرى سۈرۈش قاتارلىق تېمىلار بويىچە تۈرلۈك تەكشۈرۈپ تەتقىق قىلىشنى قانات يايدۇرۇپ، ‹‹سىياسىي كېڭەش ساداسى››نى ياڭراتتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;‹‹يېزىلاردىكى 80 –، 90 – يىللاردا تۇغۇلغانلارغا قارىتىلغان قانۇن – تۈزۈم تەربىيەسى ۋە ئىشقا ئورۇنلاشتۇرۇش مۇلازىمىتى ئۇزاق ئۈنۈملۈك خىزمەت مېخانىزمىنى ئورنىتىپ، جەمئىيەتنىڭ ئەبەدىي ئەمىنلىكىگە تەسىر يەتكۈزۈۋاتقان يوشۇرۇن ئامىللارنى مەنبەدىن تۈگىتىش كېرەك››؛ ‹‹جەنۇبىي شىنجاڭدىكى ئاز سانلىق مىللەت ئاممىنىڭ مەدەنىيەت ساپاسى ۋە تەربىيەلىنىش دەرىجىسىنى ئۆستۈرۈپ، توققۇز يىللىق مەجبۇرىيەت مائارىپىنى دەل جايىدا ئەمەلىيلەشتۈرۈش، جەنۇبىي شىنجاڭدىكى ئۈچ ۋىلايەت، بىر ئوبلاستتا 12 يىللىق ھەقسىز مەجبۇرىيەت مائارىپىنى تېز ئەمەلىيلەشتۈرۈش كېرەك››؛ ‹‹شىنجاڭنىڭ تېررورلۇققا قارشى تۇرۇش كۈرىشىنىڭ ئالاھىدىلىكىگە ماس كېلىدىغان زوراۋانلىق، تېررورلۇق جىنايى قىلمىشىغا قاقشاتقۇچ زەربە بېرىدىغان يەرلىك نىزام بەلگىلىمىلەرنى تۈزۈش كېرەك››... بۇ بىر قاتار تەكلىپلەر ئومۇمىيەتكە زىچ باغلانغان.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش ‹‹ئىجتىمائىي مۇقىملىق ۋە ئەبەدىي ئەمىنلىك››، ‹‹شىنجاڭنى قانۇن بويىچە ئىدارە قىلىش›› باش تېمىسىدىكى دائىمىي كومىتېت يىغىنى ئېچىپ، ئاپتونوم رايونلۇق پارتكومغا 14 پارچە تەكلىپ تاپشۇردى. ‹‹ئاپتونوم رايوننىڭ ئىتتىپاق، مۇقىم ۋەزىيىتىنى قەتئىي قوغداش توغرىسىدىكى قارار››نى چىقىرىپ، زوراۋانلىق، تېررورلۇق جىنايى ھەرىكەتلىرىگە قاقشاتقۇچ زەربە بېرىش، شىنجاڭنىڭ ئىجتىمائىي مۇقىملىقىنى قوغداش توغرىسىدا چاقىرىق چىقاردى... بۇ بىر قاتار ھەرىكەتلەر ئەمەلىيەتنىڭ سىنىقىدىن ئۆتتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش مۇقىملىققا ئالاقىدار مۇھىم مەسىلىدە ئېنىق پوزىتسىيە بىلدۈردى. يېقىنقى يىللاردىن بۇيان ئاپتونوم رايونىمىزدىكى ھەر دەرىجىلىك سىياسىي كېڭەش تەشكىلاتلىرى ئوخشىمىغان شەكىلدە جەمئىي 5400 قېتىمدىن كۆپرەك سۆھبەت يىغىنى، لېكسىيە، شىكايەت يىغىنى ئېچىپ، ھەر مىللەت ئاممىنى مىللىي بۆلگۈنچىلىك ۋە دىن ئەسەبىيلىكى ئىدىيەسىنىڭ سىڭىپ كىرىشىگە قەتئىي قارشى تۇرۇشقا يېتەكلەپ ۋە تەربىيەلەپ، جەمئىيەتتىكى ھەر ساھەدىكىلەرنىڭ كۈچلۈك ئىنكاسىنى قوزغىدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;شۇنىڭ بىلەن بىللە ھەر دەرىجىلىك سىياسىي كېڭەش تەشكىلاتلىرى ۋە سىياسىي كېڭەش ئەزالىرى ‹‹بەش ئاچقۇچ››تىن ئۇنىۋېرسال پايدىلىنىپ، توغرا ئېتىقاد ئارقىلىق سىقىپ چىقىرىش، مەدەنىيەت ئارقىلىق سۇسلاشتۇرۇش، قانۇن ئىدارە قىلىش ئارقىلىق چەكلەش، پەننى ئومۇملاشتۇرۇشتىن ئىبارەت ‹‹تۆتنى تەڭ باشقۇرۇش››تا چىڭ تۇرۇپ، ‹‹لېكسىيە ئۆمىكى››، ‹‹تەشۋىقاتچى››لارنىڭ رولىدىن تولۇق پايدىلىنىپ، دىن ئەسەبىيلىكى ئىدىيەسىنىڭ زىيىنىنى كەڭ تەشۋىق قىلىپ، ئاساسىي ئېقىم ئاۋازىنى ئاساسىي قاتلامغا يەتكۈزۈپ، كىشىلەرنى ياخشى تەربىيەلەپ، قەلبلەرنى تەسىرلەندۈرۈپ، ‹‹ئەسەبىيلىكنى تۈگىتىش››نى ئىلگىرى سۈرۈش داۋامىدا سىياسىي كېڭەش بىرلىك سېپىنىڭ ئاممىۋى خىزمەتتىكى ئەۋزەللىكى ۋە رولىنى جارى قىلدۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom"><strong>&nbsp;&nbsp;&nbsp;&nbsp;ماھىرلىق بىلەن ئىزدىنىپ، دادىل ئەمەلىيەتتىن ئۆتكۈزۈپ، ئەمەلىي ئۈنۈمگە ئەھمىيەت بېرىپ، كېڭىشىش شەكلىدە يېڭىلىق يارىتىش</strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;23 – مارت چۈشتىن بۇرۇن ئاپتونوم رايونلۇق سىياسىي كېڭەش بىر مەيدان قىزغىن سۆھبەت يىغىنى ئۆتكۈزدى. يىغىنغا قاتناشقان سىياسىي كېڭەش ئەزالىرى، مۇتەخەسسىسلەر، يەتتە فۇنكسىيەلىك تارماق ۋە بىر قىسىم ئاساسىي قاتلام سىياسىي كېڭەشنىڭ مەسئۇللىرى ‹‹ئەنئەنىۋى يېزا – قىشلاقلارنى قوغداش›› دېگەن تېمىنى چۆرىدەپ بەس – بەستە پىكىر بايان قىلدى. بۇ ئاپتونوم رايونلۇق سىياسىي كېڭەش 2014 – يىلى ئايلىق كېڭىشىش سۆھبەت يىغىنى ئۆتكۈزۈشنى يولغا قويغاندىن بۇيان ئۆتكۈزۈلگەن 19 – قېتىملىق ئايلىق كېڭىشىش سۆھبەت يىغىنى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;يېقىنقى يىللاردىن بۇيان ئاپتونوم رايونلۇق سىياسىي كېڭەش دىققەتنى ‹‹تۆت ئومۇميۈزلۈك›› ئىستراتېگىيەلىك ئورۇنلاشتۇرمىسىغا مەركەزلەشتۈرۈپ، ئاپتونوم رايوننىڭ مۇھىم تەدبىر ئورۇنلاشتۇرمىسىنى ئەمەلىيلەشتۈرۈشكە تۈرتكە بولۇشنى چۆرىدەپ، كېڭىشىش پائالىيىتىنىڭ ئۈنۈملۈك شەكلى ئۈستىدە پائال ئىزدىنىپ، پارتىيە، ھۆكۈمەت تارماقلىرى بىلەن بولغان خىزمەت ئالاقىسى تۈزۈمىنى يەنىمۇ مۇكەممەللەشتۈرۈپ، ئوخشىمىغان شەكىلدە مەخسۇس كېڭىشىش، نىشانلىق كېڭىشىش، ساھەلەر بويىچە كېڭىشىش، تەكلىپ بېجىرىش ئەھۋالى بويىچە كېڭىشىشنى قانات يايدۇرۇپ، كېڭىشىش ئۈنۈمىنى يەنىمۇ ئاشۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش 2014 – يىلىدىن باشلاپ سىياسىي كېڭەشنىڭ كېڭىشىش ئاساسىدىكى دېموكراتىيە قەدىمىنى ئۈزلۈكسىز تېزلىتىش ئۈستىدە ئىزدىنىپ ۋە ئۇنى كېڭەيتىپ، ئاپتونوم رايونلۇق پارتكوم ۋە جەمئىيەتتىكى ھەر ساھەنىڭ يۈكسەك دىققىتىنى قوزغىدى ۋە تولۇق مۇئەييەنلەشتۈرۈشىگە ئېرىشتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش ‹‹ئىسلاھاتنى ئومۇميۈزلۈك چوڭقۇرلاشتۇرۇش››نى باش تېما قىلىپ، سىياسىي كېڭەش ئاپتونوم رايونلۇق 11 – نۆۋەتلىك كومىتېتىنىڭ 7 – قېتىملىق دائىمىي كومىتېت يىغىنىنى ئېچىپ، مەمۇرىيەت ئىشلىرىنى مەخسۇس كېڭىشىپ، يىپەك يولى ئىقتىساد بەلبېغى يادرولۇق رايونى قۇرۇش، مەمۇرىي تەستىق تۈزۈمى ئىسلاھاتىنى چوڭقۇرلاشتۇرۇش قاتارلىق يەتتە تەكلىپ لايىھەسىنى شەكىللەندۈرۈپ، ئىسلاھاتنى ئىلگىرى سۈرۈشتىكى تەدبىر بەلگىلەشكە مەسلىھەت بېرىش رولىنى جارى قىلدۇردى؛ ‹‹شىنجاڭنى قانۇن بويىچە ئىدارە قىلىش››نى باش تېما قىلىپ، سىياسىي كېڭەش ئاپتونوم رايونلۇق 11 – نۆۋەتلىك كومىتېتىنىڭ 10 – قېتىملىق دائىمىي كومىتېت يىغىنىنى ئېچىپ، مەمۇرىيەت ئىشلىرىنى مەخسۇس كېڭىشىپ، قانۇن ئىدارە قىلىدىغان شىنجاڭ قۇرۇش ئۈچۈن ئەقىل كۆرسەتتى؛ تەڭرىتاغ 1 – مۇزلۇقى تەبىئىي مۇھاپىزەت رايونى قۇرۇشنى مەخسۇس كېڭىشىپ، دېموكراتىك نازارەت قىلىش خىزمىتىنى كۈچەيتىش ئۈستىدە ئىزدەندى؛ ‹‹شىنجاڭدا ئىختىساسلىقلارنى تەرەققىي قىلدۇرۇش››نى باش تېما قىلىپ، سىياسىي كېڭەش ئاپتونوم رايونلۇق 11 – نۆۋەتلىك كومىتېتىنىڭ 11 – قېتىملىق دائىمىي كومىتېت يىغىنىنى ئېچىپ مەمۇرىيەت ئىشلىرىنى مەخسۇس كېڭىشىپ، ئاپتونوم رايونلۇق پارتكومنىڭ ‹‹ئىختىساسلىقلار ئارقىلىق ئاپتونوم رايوننى قۇدرەت تاپقۇزۇش›› ئىستراتېگىيەلىك ئورۇنلاشتۇرمىسىنى ئەمەلىي ھەرىكىتى ئارقىلىق ئەمەلىيلەشتۈردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش 2014 – يىلى تۇنجى قېتىم كېڭىشىش خىزمىتى يىللىق پىلانىنى تۈزدى، ئىككى يىلدىن بۇيان جەمئىي 26 تۈرلۈك كېڭىشىش پىلانىنى تۈزدى، ‹‹يىللىق كېڭىشىش پىلانىنى بېكىتىش ۋە ئەمەلىيلەشتۈرۈش توغرىسىدىكى ۋاقىتلىق يولغا قويۇش چارىسى››نى تۈزۈپ، ئاپتونوم رايونلۇق پارتكوم دائىمىي كومىتېتىنىڭ تەستىقلىشى ھەم تەشكىللەپ يولغا قويۇشىغا سۇنۇپ، سىياسىي كېڭەشنىڭ كېڭىشىش پائالىيىتىنى ئاپتونوم رايونلۇق پارتكومنىڭ ئورۇنلاشتۇرمىلىرىدا تېخىمۇ نامايان قىلىپ، تۈرلۈك كېڭىشىش پائالىيەتلىرىنى تېخىمۇ قېلىپلاشتۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش 2014 – يىلى 2 – ئايدا تۇنجى قېتىملىق ئايلىق كېڭىشىش سۆھبەت يىغىنى ئاچتى، ھازىرغىچە مىللىي تېبابەت دورىگەرلىكىنى راۋاجلاندۇرۇش، سانائەت رايونلىرىنىڭ ئېنېرگىيەنى تېجەپ، بۇلغىمىنى ئازايتىشنى چۆرىدىگەن ھالدا 19 قېتىملىق ئايلىق كېڭىشىش سۆھبەت يىغىنى ئاچتى، كېڭىشىش تېمىسىنىڭ كۆپىنچىسىدە ئومۇمىيەتكە مۇناسىۋەتلىك ‹‹كىچىك مەسىلىلەر››نى چىقىش قىلىپ ئوتتۇرىغا قويغان كېڭىشىش پىكىرلىرى پارتكوم، خەلق ھۆكۈمىتى تەرىپىدىن قوبۇل قىلىنىپ، مول نەتىجە بېرىپ، سىياسىي كېڭەش خىزمىتىنىڭ بىر چوڭ يارقىن نۇقتىسىغا ئايلاندى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;2016 – يىلى چاغان تۈگىشى بىلەنلا ئاپتونوم رايونلۇق سىياسىي كېڭەش ساياھەتچىلىكنى راۋاجلاندۇرۇشنى ئىلگىرى سۈرۈش نىشانلىق كېڭىشىش سۆھبەت يىغىنى، شىنجاڭنىڭ ئالاھىدە تىرانسفورموتور ئېلېكتىر تورى قۇرۇلۇشى ۋە ئېلېكتىر ئېنېرگىيە مەخسۇس كېڭىشىش سۆھبەت يىغىنىنى ئاچتى، بۇ ئاپتونوم رايونلۇق سىياسىي كېڭەش پارتگۇرۇپپىسىنىڭ يېڭى يىلدىكى يېڭى ھەرىكىتىدە يېڭىلىق ياراتقانلىقى، شۇنداقلا كېڭىشىش ئاساسىدىكى دېموكراتىيە شەكلىنى كېڭەيتىشتىكى مۇھىم ئۇسۇل بولۇپ، سىياسىي كېڭەشنىڭ كېڭىشىش ئاساسىدىكى دېموكراتىيە قۇرۇلۇشىنى ئىلگىرى سۈرۈشتە ئۈلگىلىك رول ئوينىدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom"><strong>&nbsp;&nbsp;&nbsp;&nbsp;پۈتۈن كۈچ بىلەن ئىشلەشكە ماھىر بولۇپ، تەرەققىياتنى كۆزلەپ ئىقتىسادىي، ئىجتىمائىي ئىشلارغا مۇلازىمەت قىلىش</strong></font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ج ك پ مەركىزىي كومىتېتى سىياسىي بىيۇروسىنىڭ ئەزاسى، ئاپتونوم رايونلۇق پارتكومنىڭ شۇجىسى جاڭ چۈنشيەن 2014 – يىلىنىڭ ئاخىرىدا ‹‹شىنجاڭنى ئەڭ پاكىز جاي قىلىپ قۇرۇپ چىقىش››تەك ئېكولوگىيە مەدەنىيلىكى قۇرۇلۇشى تېمىسىنى تەنتەنىلىك ئوتتۇرىغا قويدى ھەم بۇ خىزمەتنى سىياسىي كېڭەشنىڭ مەسئۇل بولۇشىغا تاپشۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;بىر يىلدىن كېيىن ‹‹پاكىز شىنجاڭ قۇرۇش›› ئىدىيەسى كىشىلەر قەلبىدىن ئورۇن ئالدى، شىنجاڭدىكى ھەر مىللەت خەلق پاكىز شىنجاڭ قۇرۇشقا قىزغىن ئاتلاندى، شىنجاڭنىڭ تاغلىرى تېخىمۇ يېشىللىققا پۈركەندى، سۈيى تېخىمۇ سۈزۈلدى، ھاۋاسى تېخىمۇ ساپلاشتى... ئاپتونوم رايونلۇق سىياسىي كېڭەش پاكىز شىنجاڭ قۇرۇش خىزمىتىنى چىڭ تۇتۇپ، تەڭرىتاغنىڭ جەنۇبى ۋە شىمالىغا يېشىللىق ئۇرۇقىنى چاچتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;يېقىنقى يىللاردىن بۇيان ئاپتونوم رايونلۇق سىياسىي كېڭەش ‹‹تۆت ئومۇميۈزلۈك›› ئىستراتېگىيەلىك ئورۇنلاشتۇرمىسى بويىچە ئاپتونوم رايونلۇق پارتكوم ئورۇنلاشتۇرغان مەخسۇس خىزمەتلەرنى زىچ چۆرىدەپ، خەلق سىياسىي مەسلىھەت كېڭىشىنىڭ ئالاقە دائىرىسى كەڭ، ئىختىساسلىقلار توپلانغان بولۇشتەك ئەۋزەللىكىنى تىرىشىپ جارى قىلدۇرۇپ، پاكىز شىنجاڭ قۇرۇلۇشى، ئادەتتىن تاشقىرى نېفىت – گاز بايلىقىنى چارلاش – ئېچىش، خاس يېمەك – ئىچمەك كەسپىنى راۋاجلاندۇرۇش قاتارلىق خىزمەتلەردە مەسئۇلىيەتنى دادىل زىممىسىگە ئېلىپ، دادىل ئەمەلىيەتتىن ئۆتكۈزۈپ، تەشەببۇسكار بولۇپ، ئەقىل كۆرسىتىش سۈپىتىنى ئۆستۈرۈشكە كۈچەپ، سۆزى راست بولۇش، تەدبىرى ئەمەلىي بولۇش، ھەرىكىتى توغرا بولۇشنى ئىشقا ئاشۇرۇپ، سىياسىي كېڭەشنىڭ ياخشى ئاۋازىنى ياڭراتتى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش مەدەنىيەت، تارىخ ماتېرىيال خىزمىتىنى ئىلگىرى سۈرۈشتە يېڭىلىق يارىتىپ، ئاز سانلىق مىللەتلەرنىڭ 100 يىللىق ئەمەلىي خاتىرىسى، شىنجاڭنىڭ ‹‹ئۈچ ئەل›› پائالىيىتى ئەمەلىي خاتىرىسى، ياپون باسقۇنچىلىرىغا قارشى ئۇرۇش خاتىرىسى، غەربىي رايوننى كەڭ ئېچىش، ئەجدادلىرىمىز قاتارلىق تارىخىي ماتېرىياللارنى توپلاپ تۈزدى، بەزى نەشر بۇيۇملىرى ئېلىمىزنىڭ چېگرا مەدەنىيەت تارىخى كىتابلىرىنىڭ ئۈلگىسى بولدى، بەزىلىرى شىنجاڭنىڭ ياپون باسقۇنچىلىرىغا قارشى ئۇرۇش تارىخىي ماتېرىياللىرىدىكى بوشلۇقنى تولدۇرۇپ، سىياسىي كېڭەشنىڭ تارىخنى ئەستە تۇتۇپ، ئەلنى ئىدارە قىلىش، ئىتتىپاقلىشىش، ئادەم تەربىيەلەش فۇنكسىيەسىنى تولۇق جارى قىلدۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش يەنە ئىسلاھات، تەرەققىياتقا كۈچەش، خەلق تۇرمۇشىغا ئەھمىيەت بېرىش، شىنجاڭنى قانۇن بويىچە ئىدارە قىلىشتىكى ئىشلەش ۋە نەتىجە يارىتىشقا ئەھمىيەت بېرىپ، مەمۇرىيەت ئىشلىرىنى مەخسۇس كېڭىشىش ئارقىلىق ئىسلاھاتنى ئىلگىرى سۈرۈشتە تەدبىر بەلگىلەشتىن مەسلىھەت بېرىش رولىنى جارى قىلدۇردى ھەم ئاپتونوم رايونلۇق پارتكومنىڭ تۈرلۈك ئىستراتېگىيەلىك ئورۇنلاشتۇرمىلىرىنى ئەمەلىي ھەرىكىتى ئارقىلىق ئىزچىللاشتۇردى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ئاپتونوم رايونلۇق سىياسىي كېڭەش ئامما كۆڭۈل بۆلىدىغان ئىشلارنى كۆڭلىگە پۈكۈپ، سىياسىي كېڭەشنىڭ خەلق ئۈچۈن كېڭىشىش، خەلقنى كۆزلەش، خەلق ئۈچۈن ئەقىل كۆرسىتىش، خەلق ئۈچۈن كۈچ قوشۇشىنى ئاڭلىق ئىشقا ئاشۇردى. يېقىنقى يىللاردىن بۇيان ئاپتونوم رايونلۇق پارتكومنىڭ ‹‹خەلق تۇرمۇشى ئەلا، ئامما بىرىنچى، ئاساسىي قاتلام مۇھىم›› تەلىپىنى ئەستايىدىل ئىزچىللاشتۇرۇپ، خەلق ئۈچۈن ۋەزىپىسىنى ئادا قىلىشتا چىڭ تۇرۇپ، ئىشقا ئورۇنلاشتۇرۇش، مائارىپ، داۋالاش، ئىجتىمائىي كاپالەت قاتارلىق ھەر مىللەت ئاممىنىڭ مەنپەئىتىگە مۇناسىۋەتلىك مەسىلىلەرنى زىچ چۆرىدەپ پائال ئەقىل كۆرسەتتى. قىز ئىستۇدېنتلارنى ئىشقا ئورۇنلاشتۇرۇش، كەسپىي مائارىپ ۋە كەسپىي تېخنىكا ماھارىتى بويىچە تەربىيەلەشنى كۈچەيتىش، ئۈرۈمچىنىڭ ئاتموسفېرا بۇلغىنىشىنى تىزگىنلەش، يېزىلاردىكى ئارتۇق ئەمگەكچىلەرنى يۆتكەشنى تېزلىتىش ۋە يېزا يۇقىرى ئۈنۈملۈك سۇ تېجەش قۇرۇلۇشى قاتارلىق قارالمىلارنى چوڭقۇر تەكشۈرۈپ تەتقىق قىلىپ، خەلق تۇرمۇشى قۇرۇلۇشىنى ئىلگىرى سۈرۈش ئۈچۈن ئەمەلىي تەدبىر كۆرسىتىپ، ئەمەلىيلەشتۈرۈشكە تۈرتكە بولدى. قىيىنچىلىقى بار ھەر مىللەت ئاممىنىڭ ئىشلەپچىقىرىش ۋە تۇرمۇشىغا كۆڭۈل بۆلۈپ ئوتتۇرىغا قويغان قوي، كالا گۆشى تولۇقلىمىسى بېرىش سالمىقىنى ئاشۇرۇش، چېگرا رايوندىكى ئاز سانلىق مىللەت رايونلىرىنىڭ ئەمىن تاپقۇزۇپ خەلقنى بېيىتىش، ئولتۇراقلاشتۇرۇپ چارۋىچىلىقنى گۈللەندۈرۈش قۇرۇلۇشى، چارۋىچىلارنىڭ يايلاق ھوقۇق – مەنپەئىتىنى قوغداشنى كۈچەيتىش جەھەتلەردىكى تەكلىپلىرىنىڭ ھەممىسى ئاپتونوم رايونلۇق پارتكوم تەرىپىدىن قوبۇل قىلىنىپ ئەمەلىيلەشتۈرۈلدى.</font></p>
<p style="TEXT-JUSTIFY: distribute; TEXT-ALIGN: justify; FONT-FAMILY: ALKATIP Kitab Tom" align="justify"><font face="ALKATIP Kitab Tom">&nbsp;&nbsp;&nbsp;&nbsp;ھەر مىللەت، ھەر ساھە سىياسىي كېڭەش ئەزالىرى شىنجاڭنىڭ ئىقتىسادىي، ئىجتىمائىي ئىشلىرىدىكى خۇشاللىنارلىق ئۆزگىرىشلەرنى بىۋاسىتە كۆردى ۋە ئۇنىڭغا قاتناشتى، ئاپتونوم رايونلۇق سىياسىي كېڭەش ۋە سىياسىي كېڭەشكە قاتناشقان پارتىيە – گۇرۇھلار، تەشكىلاتلار، ھەر مىللەت، ھەر ساھە زاتلار شىنجاڭدىكى ھەر مىللەت خەلق بىلەن مۈرىنى – مۈرىگە تىرەپ ئىلگىرىلەپ، شانلىق نەتىجە ۋە شانلىق سەھىپە ياراتتى.</font></p><!--/enpcontent--><!--/enpcontent-->
</div>
<div id="naxir">
<p> ئەسكەرتىش: </p>
تورىمىزدىكى ئەسەرلەرنىڭ نەشىر ھوقۇقى شىنجاڭ ئۇيغۇر ئاپتونوم رايونلۇق ئاخبارات ئىشخانىسىغا تەۋە، ھەرقايسى تاراتقۇلار كۆچۈرۈپ تارقاتقاندا «تەڭرىتاغ تورى›› دەپ مەنبەنى ئېنىق ئەسكەرتىشى كېرەك، بولمىسا قانۇنىي جاۋابكارلىقى سۈرۈشتە قىلىنىدۇ.
</div>
<!-------分享链接-------->
<div id="F4" class="jiathis_style_32x32">
<span class="link"></span>
<a class="jiathis_button_weixin link01" title="分享到微信"><span class="jiathis_txt jtico jtico_weixin"></span></a>
<a class="jiathis_button_tsina link02" title="分享到新浪微博"><span class="jiathis_txt jtico jtico_tsina"></span></a>
<a class="jiathis_button_cqq link03" title="分享到QQ好友"><span class="jiathis_txt jtico jtico_cqq"></span></a>
<a class="jiathis_button_tqq link04" title="分享到腾讯微博"><span class="jiathis_txt jtico jtico_tqq"></span></a>
<a class="jiathis_button_qzone link05" title="分享到QQ空间"><span class="jiathis_txt jtico jtico_qzone"></span></a>
</div>
<script type="text/javascript" src="./orignial_webpage_files/jia.js" charset="utf-8"></script><script type="text/javascript" src="./orignial_webpage_files/plugin.client.js" charset="utf-8"></script>
</div>
<div class="ch30"></div>
<!----------------->
<div class="layout" id="F5">
<ul> <li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495431.htm" target="_blank">چاغاندا پۈتۈن مەملىكەتتىكى يۇقىرى سۈرئەتلىك تاشيوللاردا كىچىك ئاپتوموبىللاردىن ھەق ئېلىنمايدۇ</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495480.htm" target="_blank">ئاپتونوم رايوننىڭ 2015 – يىللىق خەلق ئىگىلىكى ۋە ئىجتىمائىي تەرەققىيات پىلانىنىڭ ئىجراسى ھەمدە 2016 – يىللىق خەلق ئىگىلىكى ۋە ئىجتىمائىي تەرەققىيات پىلانى لايىھەسى ھەققىدە دوكلات</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495479.htm" target="_blank">خاتالىقىنى ئۈزۈل–كېسىل تۈزىتىپ، يېڭى ھاياتقا يۈزلىنىش</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495478.htm" target="_blank">دىنىي ئەسەبىيلىك ئىدىيەسى بىلەن تەلتۆكۈس ئادا–جۇدا بولۇپ، قايتىدىن ھاياتلىققا ئېرىشىش</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495475.htm" target="_blank">شىنجاڭ دۆلەت خەۋپسىزلىكىگە خەۋپ يەتكۈزگەن 11 جىنايەتچىنىڭ جازاسىنى قانۇن بويىچە كېمەيتتى</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495461.htm" target="_blank">شىنجاڭ لياۋنىڭ كوماندىسى يېڭى پەسىلدىكى دەسلەپكى غەلىبىنى قولغا كەلتۈردى</a> </dd> </dl> </li><li><dl> <dd><a href="http://uy.ts.cn/homepage/content/2016-02/03/content_495457.htm" target="_blank">ئاپتونوم رايون مەركەزنىڭ قىشلىق، ئەتىيازلىق تۇرمۇش قۇتقۇزۇش ياردىمى مەبلىغىدىن 264 مىليون يۈەن ئاجراتتى</a> </dd> </dl> </li> </ul>
</div>
<div class="ch45"></div>
<!--------底部--------->
<div id="F6">
<iframe src="./orignial_webpage_files/node_15518.html" scrolling="no" allowtransparency="true" frameborder="0" width="100%" height="200"></iframe>
</div>
<div style="display:none">
<script type="text/javascript">var _pre = "C2_"; document.write(unescape("%3Cscript src='http://webdig.xjts.cn:8080/webdig.js?z=1' type='text/javascript'%3E%3C/script%3E"));</script><script src="./orignial_webpage_files/webdig.js" type="text/javascript"></script>
<script type="text/javascript">wd_paramtracker("_wdxid=000000000000000000000000000000000000000000")</script>
</div>
<script>
var base_url = "http://eplus.ts.cn/a/";
$(document).ready(function(){
var cid = $("#Uytscn").attr('cid');
var url = base_url+"index.php?/muslim/get_count_uy_ts_cn/"+cid;
$.getJSON(url,function(result) {
if(result.result == 10){
$("#Uytscn").text(result.count);
}
});
});
</script>
<em style="display:none" id="Uytscn" cid="483697">207634</em>
</body></html>

2
test/testdata_uni2ascii/original_uy.txt Normal file
View File

@@ -0,0 +1,2 @@

ئاپتونوم رايوننىڭ 2010–يىلىدىكى ئاپتونوم رايونلۇق پارتكوم سىياسىي كېڭەش خىزمىتى يىغىنىدىن بۇيانقى سىياسىي كېڭەش خىزمىتى توغرىسىدا ئومۇمىي بايان

326
test/testdata_uni2ascii/qq_mail_https.txt Normal file
View File

File diff suppressed because one or more lines are too long

1
test/testdata_uni2ascii/sina_read_mail.txt Normal file
View File

@@ -0,0 +1 @@
{"result":true,"errno":0,"msg":"","data":{"from":"testyouxiang2000@sohu.com","to":"\"testyouxiang2013\" <testyouxiang2013@sina.com>","cc":"","bcc":"","date":1459416272,"subject":"weiyu_fromsohu","priority":false,"notification_to":false,"xmsgid":"","isstar":false,"size":5778,"body":"<p class=\"\" align=\"left\"><span lang=\"EN-US\">&nbsp;<\/span><\/p>\n\n<p align=\"center\" dir=\"RTL\" style=\"margin: 0cm 0cm 0.0001pt; text-align: center; text-indent: 12pt; line-height: 22.5pt; direction: rtl; unicode-bidi: embed; background-image: initial; background-attachment: initial; background-size: initial; background-origin: initial; background-clip: initial; background-position: initial; background-repeat: initial;\"><span dir=\"RTL\"><\/span><span dir=\"RTL\"><\/span><span lang=\"AR-SA\" style=\"font-size:13.0pt;font-family:&quot;Times New Roman&quot;,serif;color:#4D4D4D\"><span dir=\"RTL\"><\/span><span dir=\"RTL\"><\/span>&nbsp;<strong>\u0626\u0627\u067e\u062a\u0648\u0646\u0648\u0645 \u0631\u0627\u064a\u0648\u0646\u0646\u0649\u06ad\n2010\u2013\u064a\u0649\u0644\u0649\u062f\u0649\u0643\u0649 \u0626\u0627\u067e\u062a\u0648\u0646\u0648\u0645 \u0631\u0627\u064a\u0648\u0646\u0644\u06c7\u0642 \u067e\u0627\u0631\u062a\u0643\u0648\u0645 \u0633\u0649\u064a\u0627\u0633\u0649\u064a \u0643\u06d0\u06ad\u06d5\u0634 \u062e\u0649\u0632\u0645\u0649\u062a\u0649 \u064a\u0649\u063a\u0649\u0646\u0649\u062f\u0649\u0646 \u0628\u06c7\u064a\u0627\u0646\u0642\u0649\n\u0633\u0649\u064a\u0627\u0633\u0649\u064a \u0643\u06d0\u06ad\u06d5\u0634 \u062e\u0649\u0632\u0645\u0649\u062a\u0649 \u062a\u0648\u063a\u0631\u0649\u0633\u0649\u062f\u0627 \u0626\u0648\u0645\u06c7\u0645\u0649\u064a \u0628\u0627\u064a\u0627\u0646<\/strong><\/span><span lang=\"EN-US\" dir=\"LTR\" style=\"font-size:13.0pt;font-family:&quot;UKIJ Tuz Tom&quot;,serif;color:#4D4D4D\"><o:p><\/o:p><\/span><\/p><p align=\"center\" dir=\"RTL\" style=\"margin: 0cm 0cm 0.0001pt; text-align: center; text-indent: 12pt; line-height: 22.5pt; direction: rtl; unicode-bidi: embed; background-image: initial; background-attachment: initial; background-size: initial; background-origin: initial; background-clip: initial; background-position: initial; background-repeat: initial;\"><span lang=\"AR-SA\" style=\"font-size:13.0pt;font-family:&quot;Times New Roman&quot;,serif;color:#4D4D4D\"><strong><br><\/strong><\/span><\/p><p align=\"center\" dir=\"RTL\" style=\"margin: 0cm 0cm 0.0001pt; text-align: center; text-indent: 12pt; line-height: 22.5pt; direction: rtl; unicode-bidi: embed; background-image: initial; background-attachment: initial; background-size: initial; background-origin: initial; background-clip: initial; background-position: initial; background-repeat: initial;\"><span lang=\"AR-SA\" style=\"font-size:13.0pt;font-family:&quot;Times New Roman&quot;,serif;color:#4D4D4D\"><strong>THIS THE END<\/strong><\/span><\/p><br><br><br><hr size=\"1\"><br><div><a target=\"_blank\" href=\"http:\/\/score.mail.sohu.com\/score\/item\/146?ref=mail_tailad\"><img border=\"0\" src=\"http:\/\/ad.mail.sohu.com\/mail\/images\/score_ad_foot1_750x79.png\" > <\/a><\/div>\n<br ><br ><br >","ishtml":true,"attlist":[],"mid":"0468E2CEF23DB6E2F922C3916ACC53D99300000000000001","fid":"new","sendstatus":null,"neednotify":false}}

1
test/testdata_uni2ascii/sohu_mail_unicode.txt Normal file
View File

File diff suppressed because one or more lines are too long