gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-fqdn-plugin' into 'develop-version3'

Browse Source 
Feature fqdn plugin

See merge request MESA_framework/maat!53
This commit is contained in:
刘学利
2020-09-28 16:53:40 +08:00
parent 34de556665 b29714d006
commit 87acc1a6f6
17 changed files with 1589 additions and 376 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
inc/Maat_rule.h
View File

@@ -319,6 +319,15 @@ struct ip_address
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp);
//Return order: Longger suffix first, then fqdn with bigger index first.
int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
enum MAAT_RULE_OPT
{

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

352
src/entry/FQDN_engine.cpp Normal file
View File

@@ -0,0 +1,352 @@
/*
*
* Copyright (c) 2020
* String Algorithms Research Group
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
* National Engineering Laboratory for Information Security Technologies (NELIST)
* All rights reserved
*
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
* Last modification: 2020-09-01
*
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
* Usage for direct or indirect commercial advantage is not allowed without
* written permission from the authors.
*
*/
#include "FQDN_engine.h"
#include <math.h>
#include <ctype.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
/*************************************************************************************/
//#include <nmmintrin.h>
//#define popcnt_u64 _mm_popcnt_u64
//Use gcc builtin function to replace SSE4.2 instruction for portability
#ifdef _MSC_VER
#include <intrin.h>
#define popcnt_u64 __popcnt
#else
#define popcnt_u64 __builtin_popcountl
#endif
#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++)
struct packedRT_t
{
unsigned long long bitmap[4];
unsigned int A;
unsigned char B[4];
};
void * aligned_malloc(size_t size, size_t align)
{
void * malloc_ptr;
void * aligned_ptr;
/* Error if align is not a power of two. */
if (align & (align - 1))
{
return ((void*) 0);
}
if (align==0 || size == 0)
{
return ((void *) 0);
}
malloc_ptr = malloc (sizeof(void *) + align - 1 + size);
if (!malloc_ptr)
{
return ((void *) 0);
}
aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1));
((void **) aligned_ptr) [-1] = malloc_ptr;
return aligned_ptr;
}
void aligned_free(void * aligned_ptr)
{
if (aligned_ptr)
{
free (((void **) aligned_ptr) [-1]);
}
}
/*************************************************************************************/
struct domain_impl_t
{
unsigned int id;
int suf_match;
unsigned int len;
unsigned long long hash; /*<2A><>64λ<34><CEBB>ϣֵΨһ<CEA8><D2BB>ʾһ<CABE><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>*/
domain_impl_t * next;
void * utag;
};
class CHashTrieFQDN
{
public:
CHashTrieFQDN();
~CHashTrieFQDN();
int initialize(const struct FQDN_rule * rules, size_t n_rule);
int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result);
protected:
unsigned int rank(unsigned int h);
protected:
unsigned int m_num;
domain_impl_t * m_domains;
unsigned int m_H;
unsigned int m_max_pat_len;
unsigned long long * m_B;
packedRT_t * m_RT;
domain_impl_t ** m_matched;
unsigned char m_case_tab[256];
};
const unsigned long long ONE=1;
#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) )
#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) )
const unsigned long long A=6364136223846793005LL;
CHashTrieFQDN::CHashTrieFQDN()
{
m_num=0;
m_domains=NULL;
m_B=NULL;
m_RT=NULL;
m_matched=NULL;
FOR(c, 256) m_case_tab[c]=tolower(c);
}
CHashTrieFQDN::~CHashTrieFQDN()
{
if(m_domains!=NULL)
{
delete [] m_domains;
}
if(m_B!=NULL)
{
delete [] m_B;
}
if(m_RT!=NULL)
{
aligned_free(m_RT);
}
if(m_matched!=NULL)
{
delete [] m_matched;
}
}
int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule)
{
long long mem_bytes=0;
if(n_rule==0) return -1;
m_num=n_rule;
m_domains=new domain_impl_t[m_num];
mem_bytes+=m_num*sizeof(domain_impl_t);
unsigned int N=m_num;
m_max_pat_len=0;
FOR(k, m_num)
{
m_domains[k].id =rules[k].id;
m_domains[k].suf_match=rules[k].is_suffix_match;
m_domains[k].len=rules[k].len;
m_domains[k].next=NULL;
m_domains[k].utag=rules[k].user_tag;
FOR(j, rules[k].len)
{
if(rules[k].FQDN[j]=='.') ++N;
}
if(m_max_pat_len<rules[k].len)
{
m_max_pat_len=rules[k].len;
}
}
unsigned int w=(int)(log10((double)N)/log10(2.0)+0.5)+4;
if(w<8) w=8;
m_H=(1U<<w);
m_B=new unsigned long long[m_H>>6];
mem_bytes+=(m_H>>6)*sizeof(unsigned long long);
FOR(i, (m_H>>6)) m_B[i]=0;
m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64);
mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t);
FOR(i, (m_H>>8))
{
FOR(j, 4) m_RT[i].bitmap[j]=0;
}
FOR(k, m_num)
{
const unsigned char * pb=(const unsigned char *)rules[k].FQDN;
m_domains[k].hash=0;
for(int j=rules[k].len-1; j>=-1; --j)
{
if(j==-1 || pb[j]=='.')
{
unsigned int h=m_domains[k].hash&(m_H-1);
set_bit(m_B, h);
if(j==-1)
{
int q=h&255;
m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63));
}
}
else
{
m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]];
}
}
}
m_RT[0].A=0;
FOR(i, (m_H>>8))
{
m_RT[i].B[0]=0;
m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]);
m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]);
m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]);
m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]);
}
int tn=m_RT[m_H>>8].A;
m_matched=new domain_impl_t *[tn];
mem_bytes+=tn*sizeof(domain_impl_t *);
FOR(i, tn) m_matched[i]=NULL;
FOR(k, m_num)
{
unsigned int h=m_domains[k].hash&(m_H-1);
unsigned idx=rank(h);
m_domains[k].next=m_matched[idx];
m_matched[idx]=&(m_domains[k]);
}
// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20));
return 1;
}
unsigned int CHashTrieFQDN::rank(unsigned int h)
{
int p=(h>>8);
int r=((h&255)>>6);
int s=(h&63);
unsigned long long e=m_RT[p].bitmap[r]&((ONE<<s)-1);
return m_RT[p].A+m_RT[p].B[r]+popcnt_u64(e);
}
int CHashTrieFQDN::search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result)
{
if(m_num==0 || FQDN_len==0 || FQDN==NULL || n_result==0) return -1;
size_t match_num=0;
const unsigned char * pb=(unsigned char *)FQDN;
unsigned long long HASH[16]; /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>16*/
unsigned int P[16];
int t=0;
unsigned long long hash=0;
for(int j=FQDN_len-1; j>=-1; --j)
{
if(j==-1 || pb[j]=='.')
{
unsigned int h=hash&(m_H-1);
if(is_bit_set(m_B, h)==0) break;
HASH[t]=hash;
P[t]=j+1;
++t;
}
else if(j+m_max_pat_len<FQDN_len) break;
else
{
hash=A*hash+m_case_tab[pb[j]];
}
}
for(--t; t>=0; t--)
{
unsigned int h=HASH[t]&(m_H-1);
int q=h&255;
if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63)))
{
unsigned idx=rank(h);
for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next)
{
if(P[t]!=0 && pt->suf_match==0) continue;
if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t])
{
//if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num;
results[match_num].id=pt->id;
results[match_num].offset=P[t];
results[match_num].user_tag=pt->utag;
++match_num;
if(match_num==n_result) return match_num;
}
}
}
}
return match_num;
}
/*************************************************************************************/
struct FQDN_engine
{
CHashTrieFQDN ht;
};
struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule)
{
struct FQDN_engine * instance=new struct FQDN_engine;
if(instance->ht.initialize(rules, n_rule)<0)
{
delete instance;
return NULL;
}
else
{
return instance;
}
}
int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result)
{
if(instance==NULL) return -1;
return instance->ht.search(FQDN, FQDN_len, results, n_result);
}
void FQDN_engine_free(struct FQDN_engine * instance)
{
if(instance!=NULL) delete instance;
}

157
src/entry/Maat_api.cpp
View File

@@ -1125,13 +1125,30 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
struct Maat_table_schema *table_desc=NULL;
struct Maat_table_runtime *table_rt=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return NULL;
}
table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return exdata;
}
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
@@ -1163,6 +1180,35 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
return 0;
}
int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
long argl, void *argp)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=-1;
struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
pthread_mutex_lock(&(_feather->background_update_mutex));
ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
if(ret<0)
{
pthread_mutex_unlock(&(_feather->background_update_mutex));
return -1;
}
struct Maat_table_runtime* table_rt=NULL;
if(_feather->scanner!=NULL)
{
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
}
pthread_mutex_unlock(&(_feather->background_update_mutex));
return 0;
}
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
@@ -1170,11 +1216,18 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN)
{
@@ -1190,9 +1243,57 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
ipv6_ntoh(ip_data.ipv6);
}
n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return n_get;
}
int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
struct timespec start,end;
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(_feather->scanner==NULL)
{
return 0;
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
return -1;
}
n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
}
return n_get;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
@@ -1227,7 +1328,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
return 0;
}
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1365,7 +1466,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_mid->scan_cnt++;
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virtual_table_id);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1467,7 +1568,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_mid->scan_cnt++;
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1531,7 +1632,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
return compile_ret;
}
static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_CHILD_TYPE child_type, int table_id)
static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id)
{
memset(scan_data, 0, sizeof(scan_data_t));
scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0);
@@ -1542,15 +1643,15 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
scan_data->ipv4_data.proto=proto;
switch(child_type)
{
case CHILD_TABLE_TYPE_SOURCE_IP:
case COMPONENT_TABLE_TYPE_SOURCE_IP:
scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->source);
break;
case CHILD_TABLE_TYPE_DESTINATION_IP:
case COMPONENT_TABLE_TYPE_DESTINATION_IP:
scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->dest);
break;
case CHILD_TABLE_TYPE_SESSION:
case COMPONENT_TABLE_TYPE_SESSION:
scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->source);
scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr);
@@ -1566,17 +1667,17 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
scan_data->ipv6_data.proto=proto;
switch(child_type)
{
case CHILD_TABLE_TYPE_SOURCE_IP:
case COMPONENT_TABLE_TYPE_SOURCE_IP:
memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->source);
break;
case CHILD_TABLE_TYPE_DESTINATION_IP:
case COMPONENT_TABLE_TYPE_DESTINATION_IP:
memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->dest);
break;
case CHILD_TABLE_TYPE_SESSION:
case COMPONENT_TABLE_TYPE_SESSION:
memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->source);
@@ -1596,7 +1697,7 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
}
return 0;
}
static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, Maat_table_schema* parent_table, enum MAAT_TABLE_CHILD_TYPE child_type,
static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type,
scan_result_t *region_result, unsigned int result_num,
int* virtual_table_id,
rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr,
@@ -1604,21 +1705,21 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro
{
int child_table_id=0;
if(child_type==CHILD_TABLE_TYPE_NONE)
if(child_type==COMPONENT_TABLE_TYPE_NONE)
{
child_table_id=parent_table->table_id;
child_type=CHILD_TABLE_TYPE_SESSION;
child_table_id=parent_table_id;
child_type=COMPONENT_TABLE_TYPE_SESSION;
}
else
{
child_table_id=Maat_table_get_child_id(parent_table, child_type);
child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type);
}
if(child_table_id<0)
{
return 0;
}
Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, TABLE_TYPE_IP, virtual_table_id);
Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if(real_table==NULL)
{
return 0;
@@ -1659,7 +1760,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
int region_ret=0, compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL;
scan_result_t *region_result=NULL;
Maat_table_schema* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL;
struct timespec start,end;
@@ -1670,8 +1770,9 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_mid=grab_mid(mid, _feather, thread_num, 0);
_mid->scan_cnt++;
int virtual_table_id=0;
p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
if(p_table==NULL)
enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID;
table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id);
if(table_type==TABLE_TYPE_INVALID)
{
_feather->scan_err_cnt++;
return -1;
@@ -1690,12 +1791,12 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner, thread_num);
if(p_table->table_type==TABLE_TYPE_COMPOSITION)
if(table_type==TABLE_TYPE_COMPOSITION)
{
enum MAAT_TABLE_CHILD_TYPE childs[3]={CHILD_TABLE_TYPE_SOURCE_IP, CHILD_TABLE_TYPE_DESTINATION_IP, CHILD_TABLE_TYPE_SESSION};
enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION};
for(int i=0; i<3; i++)
{
region_ret=IP_composition_scan(addr, proto, p_table, childs[i],
region_ret=IP_composition_scan(addr, proto, table_id, childs[i],
region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id,
my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
if(region_ret<0)
@@ -1713,7 +1814,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
}
else
{
region_ret=IP_composition_scan(addr, proto, p_table, CHILD_TABLE_TYPE_NONE,
region_ret=IP_composition_scan(addr, proto, table_id, COMPONENT_TABLE_TYPE_NONE,
region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id,
my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
if(region_ret<0)
@@ -1733,7 +1834,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_mid=grab_mid(mid, _feather, thread_num, 1);
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_hit_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
if(p_table->table_type==TABLE_TYPE_COMPOSITION)
if(table_type==TABLE_TYPE_COMPOSITION)
{
region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id;
}
@@ -1783,7 +1884,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
assert(thread_num<_feather->scan_thread_num);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -2071,7 +2172,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id);
p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;

51
src/entry/Maat_command.cpp
View File

@@ -201,6 +201,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
break;
case TABLE_TYPE_PLUGIN:
case TABLE_TYPE_IP_PLUGIN:
case TABLE_TYPE_FQDN_PLUGIN:
if(valid_column_seq<0)
{
return -1;
@@ -1624,7 +1625,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
struct Maat_table_schema* table_desc=NULL;
enum MAAT_SCAN_TYPE scan_type;
struct Maat_table_schema* table_schema=NULL;
void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1713,26 +1715,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
}
table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
if(rule_list[i].op==MAAT_OP_DEL)
{
if(table_type==TABLE_TYPE_PLUGIN||table_type==TABLE_TYPE_IP_PLUGIN)
{
table_desc=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL);
if(table_type==TABLE_TYPE_PLUGIN)
{
valid_column=table_desc->plugin.valid_flag_column;
}
else
{
valid_column=table_desc->ip_plugin.valid_flag_column;
}
}
else
{
valid_column=-1;
{
scan_type=Maat_table_get_scan_type(table_type);
table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL);
valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema);
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invalidate line failed, invaid format %s ."
,rule_list[i].table_line);
continue;
@@ -1918,22 +1908,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
ret=-1;
goto error_out;
}
int valid_flag_column=0;
int valid_flag_column=0;
valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table);
if(valid_flag_column<0)
{
case TABLE_TYPE_PLUGIN:
valid_flag_column=p_table->plugin.valid_flag_column;
plugin_desc=&(p_table->plugin);
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column=p_table->ip_plugin.valid_flag_column;
break;
default:
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
, line_rule[i]->rule_id
, line_rule[i]->table_name);
ret=-1;
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
, line_rule[i]->rule_id
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}

6
src/entry/Maat_ex_data.cpp
View File

@@ -57,7 +57,7 @@ void cache_row_free(void*p)
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->hash_key2ex=NULL;
@@ -207,3 +207,7 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
{
return ex_container->user_data;
}
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
{
return HASH_COUNT(ex_rt->hash_key2ex);
}

71
src/entry/Maat_rule.cpp
View File

@@ -34,7 +34,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_3_0_20200917=1;
int MAAT_FRAME_VERSION_3_0_20200927=1;
int is_valid_table_name(const char* str)
{
@@ -1392,6 +1392,8 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
scanner->gie_update_q_size++;
return 0;
}
int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger)
{
int i=0;
@@ -2163,7 +2165,6 @@ error_out:
digest_rule=NULL;
return;
}
void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
@@ -2210,21 +2211,34 @@ void update_plugin_table(struct Maat_table_schema* table_schema, const char* row
}
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
char* copy=NULL;
size_t accept_tag_offset=0, accept_tag_len=0;
if(ip_plugin_schema->rule_tag_column>0&&n_tags>0)
int rule_tag_column=-1;
if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
{
ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len);
rule_tag_column=table_schema->ip_plugin.rule_tag_column;
}
else if(table_schema->table_type==TABLE_TYPE_FQDN_PLUGIN)
{
rule_tag_column=table_schema->fqdn_plugin.rule_tag_column;
}
else
{
assert(0);
return;
}
if(rule_tag_column>0&&n_tags>0)
{
ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, could not locate tag in column %d of plugin table_schema %s:%s",
ip_plugin_schema->rule_tag_column,
"update error, could not locate tag in column %d of table %s:%s",
rule_tag_column,
table_schema->table_name[table_schema->updating_name],
table_row);
table_schema->udpate_err_cnt++;
@@ -2254,8 +2268,15 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char*
return;
}
}
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
scanner->ip_plugin_update_q_size++;
if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
{
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
}
else //TABLE_TYPE_FQDN_PLUGIN
{
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
}
scanner->xx_plugin_update_q_size++;
return;
}
@@ -2263,9 +2284,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
{
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct ip_matcher* old_ip_matcher=NULL;
int i=0, ret=0;
struct ip_matcher* old_ip_matcher=NULL;
struct FQDN_engine* old_fqdn_engine=NULL;
if(scanner->to_update_compile_cnt+scanner->to_update_group_cnt>0)
{
ret=Maat_hierarchy_rebuild(scanner->hier);
@@ -2299,8 +2320,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
switch(table_rt->table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_SIMILARITY:
ret=Maat_table_runtime_digest_batch_udpate(table_rt);
if(ret<0)
{
@@ -2308,7 +2328,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
"GIE_update error.");
}
break;
case TABLE_TYPE_IP_PLUGIN:
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
if(ret)
{
@@ -2319,6 +2339,18 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
}
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
if(ret)
{
old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt);
if(old_fqdn_engine)
{
Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
}
}
break;
default:
break;
}
@@ -2335,7 +2367,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
scanner->gie_update_q_size=0;
scanner->to_update_group_cnt=0;
scanner->to_update_compile_cnt=0;
scanner->ip_plugin_update_q_size=0;
scanner->xx_plugin_update_q_size=0;
return;
}
@@ -2400,7 +2432,7 @@ void maat_finish_cb(void* u_para)
feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size;
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size;
if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{
do_scanner_update(feather->scanner,
@@ -2480,8 +2512,9 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_IP_PLUGIN:
update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_FQDN_PLUGIN:
update_xx_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
default:
break;

586
src/entry/Maat_table.cpp
View File

@@ -18,7 +18,45 @@ struct Maat_table_manager
MESA_htable_handle map_tablename2id;
int active_plugin_table_num;
int is_last_plugin_table_updating;
void* logger;
};
enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type)
{
enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID;
switch(table_type)
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_SIMILARITY:
case TABLE_TYPE_DIGEST:
ret=SCAN_TYPE_STRING;
break;
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
ret=SCAN_TYPE_INTERVAL;
break;
case TABLE_TYPE_IP:
case TABLE_TYPE_IP_PLUS:
case TABLE_TYPE_COMPOSITION:
ret=SCAN_TYPE_IP;
break;
case TABLE_TYPE_PLUGIN:
ret=SCAN_TYPE_PLUGIN;
break;
case TABLE_TYPE_IP_PLUGIN:
ret=SCAN_TYPE_IP;
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=SCAN_TYPE_FQDN_PLUGIN;
break;
case TABLE_TYPE_COMPILE:
ret=SCAN_TYPE_NONE;
break;
default:
break;
}
return ret;
}
int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map)
{
@@ -74,25 +112,7 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA
}
return 0;
}
int read_virtual_table_schema(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map)
{
int ret=0;
char table_type[16];
ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id),
table->table_name[0],
table_type,
table->virtual_table.real_table_name);
if(ret!=4)
{
return -1;
}
ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
if(ret<0)
{
return -1;
}
return 0;
}
Maat_table_schema* table_info_new(void)
{
struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1);
@@ -119,7 +139,83 @@ int _read_integer_arrary(char* string, int *array, int size)
}
#define COLUMN_PLUGIN_SCHEMA_JSON 4
#define COLUMN_IP_PLUGIN_SCHEMA_JSON 4
#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4
#define COLUMN_COMPOSITION_SCHEMA_JSON 4
#define COLUMN_VIRUTAL_SCHEMA_JSON 4
int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map)
{
int ret=0, tmp_table_id=0;
enum MAAT_TABLE_TYPE physical_table_type=TABLE_TYPE_INVALID;
enum MAAT_SCAN_TYPE physical_table_scan_type=SCAN_TYPE_INVALID;
cJSON* json=NULL, *tmp=NULL;
char *json_str;
size_t offset=0, len=0;
char* copy_line=NULL;
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_VIRUTAL_SCHEMA_JSON, &offset, &len);
if(ret<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len]='\0';
}
json_str=copy_line+offset;
if(strchr(json_str,'{')||strchr(json_str,'['))//This is a json, mostly.
{
json=cJSON_Parse(json_str);
if(!json)
{
goto error_out;
}
if(json->type!=cJSON_Array)
{
goto error_out;
}
cJSON_ArrayForEach(tmp, json)
{
if(tmp->type!=cJSON_String)
{
goto error_out;
}
ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id);
if(ret<0)
{
goto error_out;
}
physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
if(physical_table_scan_type<SCAN_TYPE_IP)
{
goto error_out;
}
table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
}
}
else //For compatible non-json physical description
{
ret=map_str2int(table_mgr->map_tablename2id, json_str, &tmp_table_id);
if(ret<0)
{
goto error_out;
}
physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
}
cJSON_Delete(json);
free(copy_line);
return 0;
error_out:
if(json) cJSON_Delete(json);
free(copy_line);
return -1;
}
int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
{
@@ -136,7 +232,7 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
copy_line[offset+len]='\0';
}
plug_info=copy_line+offset;
@@ -173,12 +269,6 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
plugin_desc->estimate_size=tmp->valueint;
}
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
@@ -220,7 +310,7 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
copy_line[offset+len]='\0';
}
ip_plugin_info=copy_line+offset;
@@ -271,15 +361,6 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
ip_plugin_schema->estimate_size=4096;
tmp=cJSON_GetObjectItem(json, "estimate_size");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
ip_plugin_schema->estimate_size=tmp->valueint;
//read_cnt++; estimate_size is optional, so NOT ++ intentionally.
}
cJSON_Delete(json);
free(copy_line);
@@ -296,8 +377,85 @@ error_out:
return -1;
}
int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p)
{
int ret=0, read_cnt=0;
size_t offset=0, len=0;
cJSON* json=NULL, *tmp=NULL;
char* copy_line=NULL, *fqnd_plugin_schema_json=NULL;
struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin);
int read_composition_table_schema(const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
copy_line=_maat_strdup(line);
ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len);
if(ret<0)
{
goto error_out;
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len]='\0';
}
fqnd_plugin_schema_json=copy_line+offset;
json=cJSON_Parse(fqnd_plugin_schema_json);
if(!json)
{
goto error_out;
}
tmp=cJSON_GetObjectItem(json, "row_id");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->row_id_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "is_suffix_match");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "fqdn");
if(tmp!=NULL && tmp->type==cJSON_Number)
{
fqdn_plugin_schema->fqdn_column=tmp->valueint;
read_cnt++;
}
tmp=cJSON_GetObjectItem(json, "valid");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
fqdn_plugin_schema->valid_flag_column=tmp->valueint;
read_cnt++;
}
fqdn_plugin_schema->rule_tag_column=-1;
tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
fqdn_plugin_schema->rule_tag_column=tmp->valueint;
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
cJSON_Delete(json);
free(copy_line);
if(read_cnt<4)
{
return -1;
}
else
{
return 0;
}
error_out:
free(copy_line);
return -1;
}
int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
{
int ret=0;
size_t offset=0, len=0;
@@ -312,7 +470,7 @@ int read_composition_table_schema(const char* line, struct Maat_table_schema* p,
}
if(offset+len<strlen(copy_line))
{
copy_line[offset+len+1]='\0';
copy_line[offset+len]='\0';
}
composition_info=copy_line+offset;
@@ -325,126 +483,55 @@ int read_composition_table_schema(const char* line, struct Maat_table_schema* p,
tmp=cJSON_GetObjectItem(json, "source");
if(tmp!=NULL && tmp->type==cJSON_String)
{
strncpy(composition_schema->source_table.real_table_name, tmp->valuestring, sizeof(composition_schema->source_table.real_table_name));
ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP]));
if(ret<0)
{
MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
tmp->valuestring,
p->table_name[0],
p->table_id);
goto error_out;
}
}
tmp=cJSON_GetObjectItem(json, "destination");
if(tmp!=NULL && tmp->type==cJSON_String)
{
strncpy(composition_schema->destination_table.real_table_name, tmp->valuestring, sizeof(composition_schema->destination_table.real_table_name));
ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP]));
if(ret<0)
{
MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
tmp->valuestring,
p->table_name[0],
p->table_id);
goto error_out;
}
}
tmp=cJSON_GetObjectItem(json, "session");
if(tmp!=NULL && tmp->type==cJSON_String)
{
strncpy(composition_schema->session_table.real_table_name, tmp->valuestring, sizeof(composition_schema->session_table.real_table_name));
ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]));
if(ret<0)
{
MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
tmp->valuestring,
p->table_name[0],
p->table_id);
goto error_out;
}
}
cJSON_Delete(json);
free(copy_line);
return 0;
error_out:
free(copy_line);
return -1;
}
static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger)
{
struct Maat_table_schema** p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
MESA_htable_handle map_tablename2id=map_create();
size_t i=0;
int j=0, ret=0;
for(i=0;i<n_table;i++)
{
if(p_table_info[i]==NULL)
{
continue;
}
switch(p_table_info[i]->table_type)
{
case TABLE_TYPE_VIRTUAL:
ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Undefined real table %s, virtual table %s of table id %d.",
p_table_info[i]->virtual_table.real_table_name,
p_table_info[i]->table_name[0],
p_table_info[i]->table_id);
goto failed;
}
break;
case TABLE_TYPE_COMPOSITION:
if(strlen(p_table_info[i]->composition.source_table.real_table_name)>0)
{
ret=map_str2int(map_tablename2id, p_table_info[i]->composition.source_table.real_table_name,
&(p_table_info[i]->composition.source_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
p_table_info[i]->composition.source_table.real_table_name,
p_table_info[i]->table_name[0],
p_table_info[i]->table_id);
goto failed;
}
}
if(strlen(p_table_info[i]->composition.destination_table.real_table_name)>0)
{
ret=map_str2int(map_tablename2id, p_table_info[i]->composition.destination_table.real_table_name,
&(p_table_info[i]->composition.destination_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
p_table_info[i]->composition.destination_table.real_table_name,
p_table_info[i]->table_name[0],
p_table_info[i]->table_id);
goto failed;
}
}
if(strlen(p_table_info[i]->composition.session_table.real_table_name)>0)
{
ret=map_str2int(map_tablename2id, p_table_info[i]->composition.session_table.real_table_name,
&(p_table_info[i]->composition.session_table.real_table_id));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Child table %s of table %s (id=%d) are not defined.",
p_table_info[i]->composition.session_table.real_table_name,
p_table_info[i]->table_name[0],
p_table_info[i]->table_id);
goto failed;
}
}
default:
break;
}
for(j=0; j<p_table_info[i]->conj_cnt; j++)
{
ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Duplicate table %s of table id %d",
p_table_info[i]->table_name[j],
p_table_info[i]->table_id);
continue;
}
}
}
table_mgr->map_tablename2id=map_tablename2id;
return 0;
failed:
map_destroy(map_tablename2id);
return -1;
}
void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr)
{
size_t i=0;
@@ -469,7 +556,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
char line[MAX_TABLE_LINE_SIZE];
int i=0, ret=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=NULL;;
MESA_htable_handle reserved_word_map=NULL;;
struct Maat_table_schema*p=NULL;
struct Maat_table_schema*conj_table=NULL;
fp=fopen(table_info_path,"r");
@@ -483,37 +570,40 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
table_mgr=ALLOC(struct Maat_table_manager, 1);
struct Maat_table_schema** p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
table_mgr->logger=logger;
table_mgr->map_tablename2id=map_create();
string2int_map=map_create();
map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
map_register(string2int_map,"ip", TABLE_TYPE_IP);
map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS);
map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);
map_register(string2int_map,"group2group", TABLE_TYPE_GROUP2GROUP);
map_register(string2int_map,"group2compile", TABLE_TYPE_GROUP2COMPILE);
map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY);
map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL);
map_register(string2int_map,"composition", TABLE_TYPE_COMPOSITION);
map_register(string2int_map,"quickoff", 0);
map_register(string2int_map,"quickon", 1);
map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE);
// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
reserved_word_map=map_create();
map_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
map_register(reserved_word_map, "ip", TABLE_TYPE_IP);
map_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
map_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE);
map_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
map_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
map_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
map_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
map_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL);
map_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST);
map_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(reserved_word_map, "group", TABLE_TYPE_GROUP);
map_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
map_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
map_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY);
map_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
map_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION);
map_register(reserved_word_map, "quickoff", 0);
map_register(reserved_word_map, "quickon", 1);
map_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE);
// map_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
const char** charset_name_list=charset_get_all_name();
for(i=0;i<MAX_CHARSET_NUM;i++)
{
if(strlen(charset_name_list[i])>0)
{
map_register(string2int_map, charset_name_list[i], i);
map_register(reserved_word_map, charset_name_list[i], i);
}
else
{
@@ -521,8 +611,8 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
}
}
map_register(string2int_map,"yes", 1);
map_register(string2int_map,"no", 0);
map_register(reserved_word_map,"yes", 1);
map_register(reserved_word_map,"no", 0);
i=0;
@@ -546,7 +636,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
"Maat read table info %s line %d error: not enough column.",table_info_path,i);
continue;
}
ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type));
ret=map_str2int(reserved_word_map,str_tolower(table_type_str),(int*)&(p->table_type));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
@@ -557,7 +647,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
ret=read_expr_table_info(line, p, string2int_map);
ret=read_expr_table_info(line, p, reserved_word_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
@@ -570,9 +660,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal plugin info.", table_info_path,i);
"Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -580,29 +670,39 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_ip_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i);
"Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_COMPOSITION:
ret=read_composition_table_schema(line, p, string2int_map);
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=read_fqdn_plugin_table_schema(line, p);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal composition info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal composition info.", table_info_path,i);
"Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_COMPOSITION:
ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_VIRTUAL:
ret=read_virtual_table_schema(line, p, string2int_map);
ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i);
fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
"Maat read table info %s line %d error:illegal virtual info.", table_info_path,i);
"Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -610,12 +710,13 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=sscanf(not_care,"%[a-z0-9]",tmp_str);
if(ret>0)
{
ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
ret=map_str2int(reserved_word_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
}
if(ret!=1)
{
p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
}
break;
default:
break;
}
@@ -628,6 +729,15 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
goto invalid_table;
}
ret=map_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"Duplicate table %s of table id %d",
p->table_name[0],
p->table_id);
goto invalid_table;
}
if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
{
conj_table=p_table_info[p->table_id];
@@ -647,21 +757,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
//use goto to free the conjunctioned table_info
goto invalid_table;
}
p_table_info[p->table_id]=p;
table_mgr->table_cnt++;
continue;
invalid_table:
table_info_free(p);
p=NULL;
}
fclose(fp);
ret=Maat_table_build_map(table_mgr, logger);
if(ret<0)
{
return NULL;
}
map_destroy(string2int_map);
map_destroy(reserved_word_map);
return table_mgr;
}
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr)
@@ -747,13 +852,13 @@ struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* t
return table_mgr->p_table_info[table_id];
}
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id)
{
enum MAAT_SCAN_TYPE tab_scan_type;
struct Maat_table_schema **p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
struct Maat_table_schema *p_table=NULL, *p_real_table=NULL;
struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL;
if((unsigned int) table_id>n_table)
{
return NULL;
@@ -769,23 +874,20 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager*
}
if(p_table->table_type==TABLE_TYPE_VIRTUAL)
{
p_real_table=p_table_info[p_table->virtual_table.real_table_id];
p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]];
*virutal_table_id=table_id;
}
else
{
p_real_table=p_table;
p_physical_table=p_table;
if(virutal_table_id) *virutal_table_id=0;
}
if(p_real_table->table_type!=expect_type)
tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type);
if(tab_scan_type!=scan_type)
{
if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
(expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS))
{
return NULL;
}
return NULL;
}
return p_real_table;
return p_physical_table;
}
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
@@ -808,7 +910,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
void* u_para)
{
int idx=0;
struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
struct plugin_table_schema *plugin_desc=&(p_table->plugin);
if(p_table==NULL)
{
@@ -838,7 +940,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl
{
return NULL;
}
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return NULL;
@@ -863,7 +965,7 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
{
return -1;
}
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return -1;
@@ -970,7 +1072,40 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->ip_plugin.have_exdata=1;
return 0;
}
}
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger)
{
if(new_func==NULL || free_func==NULL || dup_func==NULL )
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
return -1;
}
if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__);
return -1;
}
if(table_schema->fqdn_plugin.have_exdata)
{
assert(0);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
return -1;
}
Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema,
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->fqdn_plugin.have_exdata=1;
return 0;
}
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;
@@ -1067,28 +1202,35 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char*
assert(i<=p_table->conj_cnt);
}
int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type)
int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type)
{
int ret=-1;
struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id);
if(p_table->table_type!=TABLE_TYPE_COMPOSITION)
{
return -1;
}
switch (type)
{
case CHILD_TABLE_TYPE_SOURCE_IP:
ret=p_table->composition.source_table.real_table_id;
break;
case CHILD_TABLE_TYPE_DESTINATION_IP:
ret=p_table->composition.destination_table.real_table_id;
break;
case CHILD_TABLE_TYPE_SESSION:
ret=p_table->composition.session_table.real_table_id;
break;
default:
assert(0);
break;
}
ret=p_table->composition.component_table_id[type];
return ret;
}
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table)
{
int valid_flag_column=-1;
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
valid_flag_column=p_table->plugin.valid_flag_column;
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column=p_table->ip_plugin.valid_flag_column;
break;
case TABLE_TYPE_FQDN_PLUGIN:
valid_flag_column=p_table->fqdn_plugin.valid_flag_column;
break;
default:
valid_flag_column=-1;
break;
}
return valid_flag_column;
}

261
src/entry/Maat_table_runtime.cpp
View File

@@ -113,6 +113,57 @@ static void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL;
return;
}
struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
{
struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1);
//Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$
if(fqdn[0]=='.')
{
fqdn++;
fqdn_len--;
}
if(fqdn[fqdn_len]=='/')
{
fqdn_len--;
}
fqdn_rule->FQDN=ALLOC(char, fqdn_len+1);
memcpy(fqdn_rule->FQDN, fqdn, fqdn_len);
fqdn_rule->len=fqdn_len;
fqdn_rule->is_suffix_match=is_suffix_match;
fqdn_rule->id=id;
return fqdn_rule;
}
void fqdn_rule_free(struct FQDN_rule* fqdn_rule)
{
assert(fqdn_rule->user_tag==NULL);
free(fqdn_rule->FQDN);
fqdn_rule->FQDN=NULL;
free(fqdn_rule);
return;
}
void _notype_fqdn_rule_free(void* p)
{
fqdn_rule_free((struct FQDN_rule*)p);
return;
}
struct xx_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void xx_plugin_ex_data_wrapper_free(void* ex_data)
{
struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
{
@@ -127,7 +178,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.estimate_size,
table_schema->plugin.ex_schema.key2index_func,
NULL);
if(table_schema->plugin.have_exdata)
@@ -136,8 +186,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
}
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.estimate_size,
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
if(table_schema->ip_plugin.have_exdata)
@@ -146,6 +195,16 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
}
table_rt->ip_plugin.bin=Maat_garbage_bin_new(0);
break;
case TABLE_TYPE_FQDN_PLUGIN:
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
_notype_fqdn_rule_free);
if(table_schema->fqdn_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
}
table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0);
break;
default:
break;
}
@@ -196,6 +255,12 @@ static void table_runtime_free(struct Maat_table_runtime* p)
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.new_ip_matcher==NULL);
break;
case TABLE_TYPE_FQDN_PLUGIN:
FQDN_engine_free(p->fqdn_plugin.fqdn_engine);
Maat_garbage_bin_free(p->fqdn_plugin.bin);
EX_data_rt_free(p->fqdn_plugin.ex_data_rt);
assert(p->fqdn_plugin.new_fqdn_engine==NULL);
break;
case TABLE_TYPE_PLUGIN:
EX_data_rt_free(p->plugin.ex_data_rt);
break;
@@ -358,6 +423,156 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr
MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
return;
}
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin);
struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t is_suffix_flag_offset=0, is_suffix_flag_len=0;
size_t row_id_offset=0, row_id_len=0;
size_t fqdn_offset=0, fqdn_len=0;
struct FQDN_rule* fqdn_rule=NULL;
int ret=0;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(fqdn_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find is_valid column %d of %s",
fqdn_plugin_schema->valid_flag_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find row id column %d of %s",
fqdn_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s",
fqdn_plugin_schema->is_suffix_flag_column, row);
return;
}
ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"fqdn_plugin EX data process error: cannot find fqdn column %d of %s",
fqdn_plugin_schema->fqdn_column, row);
return;
}
fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset));
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger);
}
else
{
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
}
}
else
{
EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row);
}
fqdn_plugin_rt->changed_flag=1;
return;
}
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{
struct FQDN_engine* new_fqdn_engine=NULL;
struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
struct EX_data_container **exc_array=NULL;
Maat_garbage_collect_routine(fqdn_rt->bin);
struct FQDN_rule* rules=NULL;
size_t rule_cnt=0, i=0;
if(!fqdn_rt->changed_flag)
{
assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin));
return 0;
}
rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array);
rules=ALLOC(struct FQDN_rule, rule_cnt);
for(i=0; i<rule_cnt; i++)
{
rules[i]=*((struct FQDN_rule *)(EX_data_container_get_user_data(exc_array[i])));
assert(rules[i].user_tag==exc_array[i]||rules[i].user_tag==NULL);
rules[i].user_tag=exc_array[i];
}
if(rule_cnt>0)
{
new_fqdn_engine=FQDN_engine_new(rules, rule_cnt);
}
fqdn_rt->new_fqdn_engine=new_fqdn_engine;
free(rules);
free(exc_array);
return 1;
}
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{
struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine;
table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
table_rt->fqdn_plugin.new_fqdn_engine=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
return old_one;
}
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
{
size_t i=0;
const char* row=NULL;
struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
for(i=0; i<EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt); i++)
{
row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i);
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt);
Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL.
return 0;
}
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
{
struct FQDN_match results[size];
int n_result=0, i=0;
if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
return -1;
}
if(!table_rt->fqdn_plugin.fqdn_engine)
{
return 0;
}
n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size);
for(i=0; i<n_result; i++)
{
ex_data_array[i]=EX_data_rt_get_EX_data_by_container(table_rt->fqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
}
return n_result;
}
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
{
long i=0,data_size=0;
@@ -459,41 +674,28 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
ip_plugin->changed_flag=0;
return 1;
}
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
return old_one;
}
struct ip_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void ip_plugin_ex_data_wrapper_free(void* ex_data)
{
struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
size_t is_valid_offset=0, valid_len=0;
size_t key_offset=0, key_len=0;
size_t row_id_offset=0, row_id_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -504,7 +706,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
ip_plugin_schema->row_id_column, row);
return;
}
ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len);
ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -523,17 +725,18 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger);
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger);
}
else
{
wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1);
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=key_len;
wrapper_for_free->key_offset=key_offset;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free);
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
free(ip_rule);
}
}
else

69
src/inc_internal/FQDN_engine.h Normal file
View File

@@ -0,0 +1,69 @@
/*
*
* Copyright (c) 2020
* String Algorithms Research Group
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
* National Engineering Laboratory for Information Security Technologies (NELIST)
* All rights reserved
*
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
* Last modification: 2020-09-01
*
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
* Usage for direct or indirect commercial advantage is not allowed without
* written permission from the authors.
*
*/
#ifndef H_FQDN_ENGINE_H
#define H_FQDN_ENGINE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stddef.h>
struct FQDN_rule
{
unsigned int id;
int is_suffix_match; /* is_suffix_match==0: exact match; is_suffix_match==1: longest suffix matching. */
size_t len;
char * FQDN; /* Non-ASCII character is allowed. */
void * user_tag; /* A transparent user tag for convenient accessing, the caller is responsible for its memory management. */
};
struct FQDN_engine;
struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule);
struct FQDN_match
{
unsigned int id;
unsigned int offset; /* offset==0 for exact matching; offset>0 for longest suffix matching. */
void * user_tag;
};
/*
*Function:
* Search FQDN in the rule base
*Paramters:
* instance[in]: Instance of FQDN engine
* FQDN[in]: FQDN for search
* FQDN_len[in]: Length of FQDN
* results[out]: An array to store matched FQDNs
* n_result[in]: Number of element in the result array
* Return:
* 0: No matched FQDN;
* >0: Number of matched FQNDs which were stored in results;
* <0: Error.
*/
int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result);
void FQDN_engine_free(struct FQDN_engine * instance);
#ifdef __cplusplus
}
#endif
#endif

3
src/inc_internal/Maat_ex_data.h
View File

@@ -3,7 +3,7 @@
struct EX_data_rt;
struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
void EX_data_rt_free(struct EX_data_rt* p);
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema);
void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row);
@@ -23,4 +23,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container);
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array);
void* EX_data_container_get_user_data(struct EX_data_container* ex_container);
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt);

12
src/inc_internal/Maat_rule_internal.h
View File

@@ -16,6 +16,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
#include "FQDN_engine.h"
#include <pthread.h>
#include <iconv.h>
#include <openssl/md5.h>
@@ -78,6 +79,15 @@ struct db_digest_rule
short confidence_degree;
int is_valid;
};
struct db_fqdn_rule
{
int region_id;
int group_id;
int is_suffix_match;
char* fqdn;
int is_valid;
};
struct Maat_rule_head
{
int config_id;
@@ -204,7 +214,7 @@ struct Maat_scanner
mcore_long_t ref_cnt;
rule_scanner_t region;
size_t gie_update_q_size;
size_t ip_plugin_update_q_size;
size_t xx_plugin_update_q_size;
size_t to_update_group_cnt;
size_t to_update_compile_cnt;

66
src/inc_internal/Maat_table.h
View File

@@ -17,6 +17,28 @@ enum USER_REGION_ENCODE
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
enum MAAT_SCAN_TYPE
{
SCAN_TYPE_INVALID=-1,
SCAN_TYPE_NONE=0,
SCAN_TYPE_PLUGIN,
SCAN_TYPE_IP_PLUGIN,
SCAN_TYPE_FQDN_PLUGIN,
SCAN_TYPE_IP,
SCAN_TYPE_INTERVAL,
SCAN_TYPE_STRING,
__SCAN_TYPE_MAX
};
enum MAAT_TABLE_COMPONENT_TYPE
{
COMPONENT_TABLE_TYPE_NONE=-1,
COMPONENT_TABLE_TYPE_SOURCE_IP=0,
COMPONENT_TABLE_TYPE_DESTINATION_IP,
COMPONENT_TABLE_TYPE_SESSION,
__COMPONENT_TABLE_TYPE_MAX
};
enum MAAT_TABLE_TYPE
{
@@ -28,6 +50,8 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
TABLE_TYPE_INTERVAL_PLUS,
//Above are physical table for scan
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_GROUP2GROUP,
@@ -36,7 +60,7 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_INTERVAL_PLUS
TABLE_TYPE_FQDN_PLUGIN
};
struct compile_ex_data_idx
@@ -68,12 +92,11 @@ struct expr_table_schema
};
struct virtual_table_schema
{
int real_table_id;
char real_table_name[MAX_TABLE_NAME_LEN];
int physical_table_id[__SCAN_TYPE_MAX];
};
struct composition_table_schema
{
struct virtual_table_schema source_table, destination_table, session_table;
int component_table_id[__COMPONENT_TABLE_TYPE_MAX];
};
struct plugin_table_callback_schema
{
@@ -100,7 +123,6 @@ struct plugin_table_schema
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
long long estimate_size;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct EX_data_schema ex_schema;
};
@@ -112,7 +134,17 @@ struct ip_plugin_table_schema
int end_ip_column;
int valid_flag_column;
int rule_tag_column;
long long estimate_size;
int have_exdata;
struct EX_data_schema ex_schema;
};
struct fqdn_plugin_table_schema
{
int row_id_column;
int is_suffix_flag_column;
int fqdn_column;
int valid_flag_column;
int rule_tag_column;
int have_exdata;
struct EX_data_schema ex_schema;
};
@@ -129,6 +161,7 @@ struct Maat_table_schema
struct expr_table_schema expr;
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
struct fqdn_plugin_table_schema fqdn_plugin;
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
void* others;//group, ip, interval and digest don't have any special schema.
@@ -144,8 +177,9 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type);
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
@@ -181,6 +215,13 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
Maat_plugin_EX_new_func_t* new_func,
Maat_plugin_EX_free_func_t* free_func,
Maat_plugin_EX_dup_func_t* dup_func,
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
@@ -188,12 +229,7 @@ void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mg
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
enum MAAT_TABLE_CHILD_TYPE
{
CHILD_TABLE_TYPE_NONE=-1,
CHILD_TABLE_TYPE_SOURCE_IP=0,
CHILD_TABLE_TYPE_DESTINATION_IP,
CHILD_TABLE_TYPE_SESSION
};
int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type);
int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table);

19
src/inc_internal/Maat_table_runtime.h
View File

@@ -4,6 +4,7 @@
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "FQDN_engine.h"
#include "alignment_int64.h"
#include "dynamic_array.h"
#include <MESA/MESA_htable.h>
@@ -14,7 +15,14 @@ struct similar_runtime
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct fqdn_plugin_runtime
{
struct FQDN_engine* fqdn_engine;
struct FQDN_engine* new_fqdn_engine;
struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY
struct Maat_garbage_bin* bin;
int changed_flag;
};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
@@ -51,6 +59,7 @@ struct Maat_table_runtime
union
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
@@ -86,4 +95,12 @@ int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt);
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);

101
test/maat_json.json
View File

@@ -60,7 +60,7 @@
"regions": [
{
"table_name": "GeoLocation",
"table_type": "string",
"table_type": "expr",
"table_content": {
"keywords": "Greece.Sparta",
"expr_type": "none",
@@ -69,6 +69,35 @@
}
}
]
},
{
"group_name": "FQDN_OBJ1",
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "sports.example.com",
"expr_type": "none",
"match_method": "exact",
"format": "uncase plain"
}
}
]
},
{
"group_name": "FQDN_CAT1",
"regions": [
{
"table_name": "INTERGER_PLUS",
"table_type": "intval_plus",
"table_content": {
"district": "fqdn_cat_id",
"low_boundary": 1724,
"up_boundary": 1724
}
}
]
}
],
"rules": [
@@ -1220,7 +1249,7 @@
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "VirtualWithPhysical",
"user_region": "VirtualWithOnePhysical",
"is_valid": "yes",
"groups": [
{
@@ -1850,51 +1879,85 @@
]
}
]
}
},
{
"compile_id": 180,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "Hierarchy_VirtualWithTwoPhysical",
"is_valid": "yes",
"groups": [
{
"group_name":"FQDN_OBJ1",
"virtual_table":"VIRTUAL_SSL_SNI",
"not_flag" : 0,
"clause_index":0
},
{
"group_name":"FQDN_CAT1",
"virtual_table":"VIRTUAL_SSL_SNI",
"not_flag" : 0,
"clause_index":0
}
]
}
],
"plugin_table": [
{
"table_name": "QD_ENTRY_INFO",
"table_content": [
"1\t192.168.0.1\t101\t1",
"2\t192.168.0.2\t101\t1",
"3\t192.168.1.1\t102\t1"
"1\t192.168.0.1\t101\t1",
"2\t192.168.0.2\t101\t1",
"3\t192.168.1.1\t102\t1"
]
},
{
"table_name": "TEST_PLUGIN_TABLE",
"table_content": [
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
]
},
{
"table_name": "TEST_EFFECTIVE_RANGE_TABLE",
"table_content": [
"1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
"2\tSUCCESS\t66\t1\t0\t222",
"3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
"4\tSUCCESS\t66\t1\t{}\t444",
"5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
"6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
"1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
"2\tSUCCESS\t66\t1\t0\t222",
"3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
"4\tSUCCESS\t66\t1\t{}\t444",
"5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
"6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
]
},
{
"table_name": "IR_INTERCEPT_IP",
"table_content": [
"1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
"161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
"1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
"161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
]
},
{
"table_name": "TEST_IP_PLUGIN_WITH_EXDATA",
"table_content": [
"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1",
"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1"
]
},
{
"table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_content": [
"201\t0\twww.example1.com\tcatid=1\t1",
"202\t1\t.example1.com\tcatid=1\t1",
"203\t0\tnews.example1.com\tcatid=2\t1",
"204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
"205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
]
}
]
}

4
test/table_info.conf
View File

@@ -55,4 +55,6 @@
32 DESTINATION_IP_ASN virtual AS_NUMBER --
33 GeoLocation expr UTF8 UTF8 yes 0
34 SOURCE_IP_GEO virtual GeoLocation --
35 INTERGER_PLUS intval_plus --
35 INTERGER_PLUS intval_plus --
36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} --
37 VIRTUAL_SSL_SNI virtual ["KEYWORDS_TABLE","INTERGER_PLUS"] --

196
test/test_maatframe.cpp
View File

@@ -310,6 +310,86 @@ TEST(IP_Plugin_Table, EX_DATA)
}
#define FQDN_PLUGIN_EX_DATA
struct fqdn_plugin_ud
{
int rule_id;
int catid;
int ref_cnt;
};
void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0;
struct fqdn_plugin_ud* ud=(struct fqdn_plugin_ud*)calloc(sizeof(struct fqdn_plugin_ud), 1);
ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id=atoi(table_line+column_offset);
ret=Maat_helper_read_column(table_line, 4, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
sscanf(table_line+column_offset, "catid=%d",&ud->catid);
ud->ref_cnt=1;
*ad=ud;
(*counter)++;
return;
}
void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{
struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return;
free(u);
*ad=NULL;
}
void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{
struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*from);
u->ref_cnt++;
*to=u;
}
TEST(FQDN_Plugin_Table, EX_DATA)
{
int fqdn_plugin_ex_data_counter=0, i=0;
const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
int table_id=0, ret=0;
table_id=Maat_table_register(g_feather, table_name);
ASSERT_GT(table_id, 0);
ret=Maat_fqdn_plugin_EX_register(g_feather, table_id,
fqdn_plugin_EX_new_cb,
fqdn_plugin_EX_free_cb,
fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
struct fqdn_plugin_ud* result[4];
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example1.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 201);
EXPECT_EQ(result[1]->rule_id, 202);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example3.com", (void**)result, 4);
EXPECT_EQ(ret, 0);
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
EXPECT_EQ(result[0]->rule_id, 205);
EXPECT_EQ(result[1]->rule_id, 204);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
}
TEST(StringScan, Full)
{
int ret=0;
@@ -1755,9 +1835,9 @@ TEST(ScanResult, LongerServiceDefine)
free(buff);
return;
}
TEST(Hierarchy, VirtualWithPhysical)
TEST(Hierarchy, VirtualOfOnePhysical)
{
#define Hierarchy_VirtualWithPhysical
#define Hierarchy_VirtualOfOnePhysical
int ret=0, table_id=0;
const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058";
@@ -1796,6 +1876,42 @@ TEST(Hierarchy, VirtualWithPhysical)
return;
}
TEST(Hierarchy, VirtualOfTwoPhysical)
{
#define Hierarchy_VirtualWithTwoPhysical
int ret=0, table_id=0;
const char* sni="sports.example.com";
int cat_id=1724;
struct Maat_rule_t result[4];
memset(result, 0, sizeof(result));
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather, "VIRTUAL_SSL_SNI");
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, sni, strlen(sni),
result, NULL, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 180);
Maat_clean_status(&mid);
memset(result, 0, sizeof(result));
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "fqdn_cat_id", strlen("fqdn_cat_id"));
ASSERT_EQ(ret, 0);
ret=Maat_scan_intval(g_feather, table_id, cat_id, result,4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 180);
Maat_clean_status(&mid);
return;
}
TEST(Hierarchy, VirtualWithVirtual)
{
#define Hierarchy_VirtualWithVirtual
@@ -1836,6 +1952,7 @@ TEST(Hierarchy, VirtualWithVirtual)
return;
}
TEST(Hierarchy, OneGroupInTwoVirtual)
{
#define Hierarchy_OneGroupInTwoVirtual
@@ -3110,9 +3227,9 @@ TEST_F(MaatCmdTest, PluginEXData)
return;
}
TEST_F(MaatCmdTest, IPPluginEXData)
TEST_F(MaatCmdTest, UpdateIPPlugin)
{
#define ip_plugin_EX_data
#define IP_Plugin_EX_data
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
@@ -3178,7 +3295,7 @@ TEST_F(MaatCmdTest, IPPluginEXData)
ip_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_cmd_set_lines(feather, p_line , TEST_CMD_LINE_NUM, MAAT_OP_DEL);
ret=Maat_cmd_set_lines(feather, p_line, TEST_CMD_LINE_NUM, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
@@ -3188,6 +3305,75 @@ TEST_F(MaatCmdTest, IPPluginEXData)
return;
}
TEST_F(MaatCmdTest, UpdateFQDNPlugin)
{
#define FQDN_Plugin_EX_data
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
int table_id=0, fqdn_plugin_ex_data_counter=0;
const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
const int TEST_CMD_LINE_NUM=5;
const struct Maat_cmd_line *p_line[TEST_CMD_LINE_NUM];
struct Maat_cmd_line line_rule[TEST_CMD_LINE_NUM];
const char* table_line[TEST_CMD_LINE_NUM]={
"201\t0\twww.example1.com\tcatid=1\t1",
"202\t1\t.example1.com\tcatid=1\t1",
"203\t0\tnews.example1.com\tcatid=2\t1",
"204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
"205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
};
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
memset(&line_rule,0,sizeof(line_rule));
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
line_rule[i].table_name=table_name;
line_rule[i].table_line=table_line[i];
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
ret=Maat_fqdn_plugin_EX_register(feather, table_id,
fqdn_plugin_EX_new_cb,
fqdn_plugin_EX_free_cb,
fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
struct fqdn_plugin_ud* result[4];
ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 2);
for(i=0; i<ret; i++)
{
fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
ret=Maat_cmd_set_lines(feather, p_line+3, TEST_CMD_LINE_NUM-3, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US*5);
ret=Maat_fqdn_plugin_get_EX_data(feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
ASSERT_EQ(ret, 0);
return;
}
#define TEST_HIT_PATH
TEST_F(MaatCmdTest, HitPath)
{