framework work well
This commit is contained in:
liuwentan
@@ -4,7 +4,7 @@ project(maatframe)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/include)
|
||||
|
||||
set(CMAKE_C_FLAGS "-std=gnu99 -Wall")
|
||||
set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} "-Wall")
|
||||
set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall)
|
||||
|
||||
include_directories(include)
|
||||
|
||||
@@ -14,3 +14,4 @@ enable_testing()
|
||||
add_subdirectory(vendor)
|
||||
add_subdirectory(src)
|
||||
add_subdirectory(test)
|
||||
add_subdirectory(scanner)
|
||||
0
src/cJSON.c → deps/cJSON/cJSON.c
vendored
0
src/cJSON.c → deps/cJSON/cJSON.c
vendored
390
deps/log/log.c
vendored
Normal file
390
deps/log/log.c
vendored
Normal file
@@ -0,0 +1,390 @@
|
||||
/*
|
||||
* Copyright (c) 2020 rxi
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to
|
||||
* deal in the Software without restriction, including without limitation the
|
||||
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
* sell copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
* IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <pthread.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "log.h"
|
||||
|
||||
#define MAX_CALLBACKS 32
|
||||
|
||||
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
|
||||
|
||||
typedef struct {
|
||||
log_func_t fn;
|
||||
void *user_data;
|
||||
int level;
|
||||
} Callback;
|
||||
|
||||
struct log_handle
|
||||
{
|
||||
int level;
|
||||
int enable;
|
||||
void *user_data;
|
||||
log_lock_func_t lock;
|
||||
char run_log_path[1024];
|
||||
char cur_log_file[1024];
|
||||
pthread_mutex_t mutex;
|
||||
Callback callbacks[MAX_CALLBACKS];
|
||||
};
|
||||
|
||||
static unsigned char weekday_str[7][4] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
|
||||
|
||||
static unsigned char month_str[12][4] = {"Jan", "Feb", "Mar", "Apr","May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
|
||||
|
||||
static int log_create_dir(const char *dir_path, int path_len)
|
||||
{
|
||||
if(dir_path == NULL)
|
||||
return -1;
|
||||
|
||||
char *buf = (char *)calloc(path_len+1, 1);
|
||||
int ret = -1;
|
||||
|
||||
memcpy(buf, dir_path, path_len);
|
||||
if(access(buf, R_OK) != 0)
|
||||
{
|
||||
if(mkdir(buf, 0755)!= 0)
|
||||
ret = -1;
|
||||
else
|
||||
ret = 0;
|
||||
}
|
||||
else
|
||||
ret = 1;
|
||||
free(buf);
|
||||
buf = NULL;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void log_close_file(struct log_event *event)
|
||||
{
|
||||
pthread_mutex_lock(&event->mutex);
|
||||
FILE *fp=(FILE *)event->user_data;
|
||||
if(fp != NULL)
|
||||
{
|
||||
fclose(fp);
|
||||
event->user_data = NULL;
|
||||
}
|
||||
pthread_mutex_unlock(&event->mutex);
|
||||
return;
|
||||
}
|
||||
|
||||
int log_open_file(char *file_name, struct log_event *event)
|
||||
{
|
||||
FILE *fp = NULL;
|
||||
log_close_file(event);
|
||||
if(NULL == (fp = fopen(file_name, "a")))
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
memcpy(event->cur_log_file, file_name, strlen(file_name));
|
||||
event->user_data = fp;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int log_create_path(const char *file_path)
|
||||
{
|
||||
FILE *fp = NULL;
|
||||
|
||||
if(file_path == NULL)
|
||||
return 0;
|
||||
|
||||
char *p_path = rindex(file_path, '/');
|
||||
if(p_path==0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char *p_cur = file_path;
|
||||
int path_len = p_path - file_path;
|
||||
int i = 0;
|
||||
|
||||
if(log_create_dir(file_path, path_len) >= 0)
|
||||
return 0;
|
||||
|
||||
for(;i<=path_len;i++,p_cur++)
|
||||
{
|
||||
if(*p_cur == '/')
|
||||
{
|
||||
if(log_create_dir(file_path, i+1) < 0)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
if(NULL == (fp = fopen(file_path, "w")))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int log_create_log_file(struct log_event *event)
|
||||
{
|
||||
time_t t;
|
||||
struct tm local_time;
|
||||
char tmp_log_file_name[512];
|
||||
|
||||
FILE *fp = (FILE *)event->user_data;
|
||||
|
||||
time(&t);
|
||||
if(NULL == (localtime_r(&t, &local_time)))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
sprintf(tmp_log_file_name, "%s.%04d-%02d-%02d", event->run_log_path, local_time.tm_year + 1900, local_time.tm_mon + 1, local_time.tm_mday);
|
||||
|
||||
if(fp == NULL)
|
||||
{
|
||||
if(0 != log_open_file(tmp_log_file_name, event)) return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (0 != memcmp(tmp_log_file_name, event->cur_log_file, strlen(tmp_log_file_name)))
|
||||
{
|
||||
if(0 != log_open_file(tmp_log_file_name, event))return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void log_print_file(struct log_event *event)
|
||||
{
|
||||
char buf[64];
|
||||
time_t t;
|
||||
struct tm local_time;
|
||||
const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
|
||||
|
||||
time(&t);
|
||||
if(NULL == (localtime_r(&t, &local_time))) return;
|
||||
snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday],
|
||||
month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900);
|
||||
|
||||
log_create_log_file(event);
|
||||
fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module);
|
||||
|
||||
vfprintf((FILE *)event->user_data, event->fmt, event->ap);
|
||||
fprintf((FILE *)event->user_data, "\n");
|
||||
fflush((FILE *)event->user_data);
|
||||
}
|
||||
|
||||
static void log_print_console(struct log_event *event)
|
||||
{
|
||||
char buf[64]={0};
|
||||
time_t t;
|
||||
struct tm local_time;
|
||||
const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"};
|
||||
|
||||
time(&t);
|
||||
if(NULL == (localtime_r(&t, &local_time))) return;
|
||||
snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday],
|
||||
month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900);
|
||||
fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module);
|
||||
|
||||
vfprintf((FILE *)event->user_data, event->fmt, event->ap);
|
||||
fprintf((FILE *)event->user_data, "\n");
|
||||
fflush((FILE *)event->user_data);
|
||||
}
|
||||
|
||||
void log_options_set_lock_fn(struct log_handle * handle, log_lock_func_t fn, void *user_data)
|
||||
{
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t != NULL)
|
||||
{
|
||||
_handle_t->lock = fn;
|
||||
_handle_t->user_data = user_data;
|
||||
}
|
||||
}
|
||||
|
||||
void log_options_set_level(struct log_handle * handle, int level)
|
||||
{
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t != NULL)
|
||||
{
|
||||
_handle_t->level = level;
|
||||
}
|
||||
}
|
||||
|
||||
void log_options_set_enable(struct log_handle * handle, int enable)
|
||||
{
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t != NULL)
|
||||
{
|
||||
_handle_t->enable = enable;
|
||||
}
|
||||
}
|
||||
|
||||
int log_add_method_callback(void *handle, log_func_t fn, void *udata, int level)
|
||||
{
|
||||
int i=0;
|
||||
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t == NULL)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0; i < MAX_CALLBACKS; i++)
|
||||
{
|
||||
if (!_handle_t->callbacks[i].fn)
|
||||
{
|
||||
_handle_t->callbacks[i] = (Callback) { fn, udata, level };
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
int log_options_add_file_fp(struct log_handle *handle, int level)
|
||||
{
|
||||
int ret=0;
|
||||
FILE *fp=NULL;
|
||||
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(!_handle_t)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
ret =log_create_path(_handle_t->run_log_path);
|
||||
if(ret==0)
|
||||
{
|
||||
fp=stderr;
|
||||
}
|
||||
return log_add_method_callback(handle, log_print_file, fp, level);
|
||||
}
|
||||
|
||||
int log_options_add_fp(struct log_handle * handle, FILE *fp, int level)
|
||||
{
|
||||
return log_add_method_callback(handle, log_print_file, fp, level);
|
||||
}
|
||||
|
||||
static void log_lock(void *handle)
|
||||
{
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t != NULL && _handle_t->lock)
|
||||
{
|
||||
_handle_t->lock(1, _handle_t->user_data);
|
||||
}
|
||||
}
|
||||
|
||||
static void log_unlock(void *handle)
|
||||
{
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(_handle_t != NULL && _handle_t->lock)
|
||||
{
|
||||
_handle_t->lock(0, _handle_t->user_data);
|
||||
}
|
||||
}
|
||||
|
||||
static void log_init_event(struct log_event *event, void *user_data)
|
||||
{
|
||||
event->user_data = user_data;
|
||||
}
|
||||
|
||||
struct log_handle *log_handle_create(const char *file_path, int level)
|
||||
{
|
||||
struct log_handle *_handle_t = ALLOC(struct log_handle, 1);
|
||||
if(!_handle_t)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
_handle_t->enable=1;
|
||||
_handle_t->level = level;
|
||||
strncpy(_handle_t->run_log_path, file_path, 1024);
|
||||
pthread_mutex_init(&_handle_t->mutex,NULL);
|
||||
|
||||
log_options_add_file_fp(_handle_t, level);
|
||||
|
||||
return _handle_t;
|
||||
}
|
||||
|
||||
void log_handle_destroy(struct log_handle * handle)
|
||||
{
|
||||
int i=0;
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
if(!_handle_t)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (_handle_t->user_data != NULL)
|
||||
{
|
||||
free(_handle_t->user_data);
|
||||
_handle_t->user_data = NULL;
|
||||
}
|
||||
for(i=0; i<MAX_CALLBACKS; i++)
|
||||
{
|
||||
if(_handle_t->callbacks && _handle_t->callbacks->user_data)
|
||||
{
|
||||
fclose(_handle_t->callbacks->user_data);
|
||||
_handle_t->callbacks->user_data = NULL;
|
||||
}
|
||||
}
|
||||
pthread_mutex_destroy(&(_handle_t->mutex));
|
||||
free(handle);
|
||||
handle = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
void log_print(struct log_handle *handle, int level, const char *module, const char *fmt, ...)
|
||||
{
|
||||
struct log_event event;
|
||||
memset(&event, 0, sizeof(event));
|
||||
|
||||
struct log_handle *_handle_t = (struct log_handle *)handle;
|
||||
|
||||
event.fmt=fmt;
|
||||
event.level=level;
|
||||
event.module=module;
|
||||
event.mutex=_handle_t->mutex;
|
||||
event.cur_log_file=_handle_t->cur_log_file;
|
||||
event.run_log_path=_handle_t->run_log_path;
|
||||
|
||||
if(_handle_t->enable != 1 && level >= _handle_t->level)
|
||||
{
|
||||
log_init_event(&event, stderr);
|
||||
va_start(event.ap, fmt);
|
||||
log_print_console(&event);
|
||||
va_end(event.ap);
|
||||
}
|
||||
|
||||
log_lock(handle);
|
||||
|
||||
int i=0;
|
||||
for(i=0; i<MAX_CALLBACKS && _handle_t->callbacks[i].fn; i++)
|
||||
{
|
||||
Callback *cb = &_handle_t->callbacks[i];
|
||||
if (level >= cb->level)
|
||||
{
|
||||
log_init_event(&event, cb->user_data);
|
||||
va_start(event.ap, fmt);
|
||||
cb->fn(&event);
|
||||
va_end(event.ap);
|
||||
cb->user_data = event.user_data;
|
||||
}
|
||||
}
|
||||
|
||||
log_unlock(handle);
|
||||
}
|
||||
65
deps/log/log.h
vendored
Normal file
65
deps/log/log.h
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Copyright (c) 2020 rxi
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the MIT license. See `log.c` for details.
|
||||
*/
|
||||
|
||||
#ifndef LOG_H
|
||||
#define LOG_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdbool.h>
|
||||
#include <time.h>
|
||||
|
||||
#define LOG_VERSION "0.1.0"
|
||||
|
||||
struct log_handle;
|
||||
|
||||
struct log_event
|
||||
{
|
||||
int level;
|
||||
va_list ap;
|
||||
const char *fmt;
|
||||
const char *module;
|
||||
char *run_log_path;
|
||||
char *cur_log_file;
|
||||
void *user_data;
|
||||
pthread_mutex_t mutex;
|
||||
};
|
||||
|
||||
typedef void (*log_func_t)(struct log_event *ev);
|
||||
typedef void (*log_lock_func_t)(int lock, void *udata);
|
||||
|
||||
enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL };
|
||||
|
||||
#define log_debug(handle, module, fmt, ...) log_print(handle, LOG_DEBUG, module, fmt, ##__VA_ARGS__)
|
||||
#define log_trace(handle, module, fmt, ...) log_print(handle, LOG_TRACE, module, fmt, ##__VA_ARGS__)
|
||||
#define log_info(handle, module, fmt, ...) log_print(handle, LOG_INFO, module, fmt, ##__VA_ARGS__)
|
||||
#define log_warn(handle, module, fmt, ...) log_print(handle, LOG_WARN, module, fmt, ##__VA_ARGS__)
|
||||
#define log_error(handle, module, fmt, ...) log_print(handle, LOG_ERROR, module, fmt, ##__VA_ARGS__)
|
||||
#define log_fatal(handle, module, fmt, ...) log_print(handle, LOG_FATAL, module, fmt, ##__VA_ARGS__)
|
||||
|
||||
void log_print(struct log_handle *, int level, const char *module, const char *fmt, ...);
|
||||
|
||||
int log_options_add_file_fp(struct log_handle *, int level);
|
||||
int log_options_add_fp(struct log_handle *, FILE *fp, int level);
|
||||
|
||||
void log_options_set_lock_fn(struct log_handle *, log_lock_func_t fn, void *user_data);
|
||||
void log_options_set_enable(struct log_handle *, int enable);
|
||||
void log_options_set_level(struct log_handle *, int level);
|
||||
|
||||
struct log_handle * log_handle_create(const char *file_path, int level);
|
||||
void log_handle_destroy(struct log_handle *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
12
deps/sds/Changelog
vendored
12
deps/sds/Changelog
vendored
@@ -1,12 +0,0 @@
|
||||
Version 2.0
|
||||
===
|
||||
|
||||
* Better memory usage. A bit slower in certain workloads.
|
||||
* sdscatfmt() replacement for sdscatprintf() for speed critical code added.
|
||||
* Ability to easily switch allocator just changing sdsalloc.h
|
||||
* No longer binary compatible with SDS v1.0.
|
||||
|
||||
Version 1.0
|
||||
===
|
||||
|
||||
* Initial SDS stand alone verison.
|
||||
24
deps/sds/LICENSE
vendored
24
deps/sds/LICENSE
vendored
@@ -1,24 +0,0 @@
|
||||
Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
8
deps/sds/Makefile
vendored
8
deps/sds/Makefile
vendored
@@ -1,8 +0,0 @@
|
||||
all: sds-test
|
||||
|
||||
sds-test: sds.c sds.h testhelp.h
|
||||
$(CC) -o sds-test sds.c -Wall -std=c99 -pedantic -O2 -DSDS_TEST_MAIN
|
||||
@echo ">>> Type ./sds-test to run the sds.c unit tests."
|
||||
|
||||
clean:
|
||||
rm -f sds-test
|
||||
917
deps/sds/README.md
vendored
917
deps/sds/README.md
vendored
@@ -1,917 +0,0 @@
|
||||
Simple Dynamic Strings
|
||||
===
|
||||
|
||||
**Notes about version 2**: this is an updated version of SDS in an attempt
|
||||
to finally unify Redis, Disque, Hiredis, and the stand alone SDS versions.
|
||||
This version is **NOT* binary compatible** with SDS verison 1, but the API
|
||||
is 99% compatible so switching to the new lib should be trivial.
|
||||
|
||||
Note that this version of SDS may be a slower with certain workloads, but
|
||||
uses less memory compared to V1 since header size is dynamic and depends to
|
||||
the string to alloc.
|
||||
|
||||
Moreover it includes a few more API functions, notably `sdscatfmt` which
|
||||
is a faster version of `sdscatprintf` that can be used for the simpler
|
||||
cases in order to avoid the libc `printf` family functions performance
|
||||
penalty.
|
||||
|
||||
How SDS strings work
|
||||
===
|
||||
|
||||
SDS is a string library for C designed to augment the limited libc string
|
||||
handling functionalities by adding heap allocated strings that are:
|
||||
|
||||
* Simpler to use.
|
||||
* Binary safe.
|
||||
* Computationally more efficient.
|
||||
* But yet... Compatible with normal C string functions.
|
||||
|
||||
This is achieved using an alternative design in which instead of using a C
|
||||
structure to represent a string, we use a binary prefix that is stored
|
||||
before the actual pointer to the string that is returned by SDS to the user.
|
||||
|
||||
+--------+-------------------------------+-----------+
|
||||
| Header | Binary safe C alike string... | Null term |
|
||||
+--------+-------------------------------+-----------+
|
||||
|
|
||||
`-> Pointer returned to the user.
|
||||
|
||||
Because of meta data stored before the actual returned pointer as a prefix,
|
||||
and because of every SDS string implicitly adding a null term at the end of
|
||||
the string regardless of the actual content of the string, SDS strings work
|
||||
well together with C strings and the user is free to use them interchangeably
|
||||
with other std C string functions that access the string in read-only.
|
||||
|
||||
SDS was a C string I developed in the past for my everyday C programming needs,
|
||||
later it was moved into Redis where it is used extensively and where it was
|
||||
modified in order to be suitable for high performance operations. Now it was
|
||||
extracted from Redis and forked as a stand alone project.
|
||||
|
||||
Because of its many years life inside Redis, SDS provides both higher level
|
||||
functions for easy strings manipulation in C, but also a set of low level
|
||||
functions that make it possible to write high performance code without paying
|
||||
a penalty for using an higher level string library.
|
||||
|
||||
Advantages and disadvantages of SDS
|
||||
===
|
||||
|
||||
Normally dynamic string libraries for C are implemented using a structure
|
||||
that defines the string. The structure has a pointer field that is managed
|
||||
by the string function, so it looks like this:
|
||||
|
||||
```c
|
||||
struct yourAverageStringLibrary {
|
||||
char *buf;
|
||||
size_t len;
|
||||
... possibly more fields here ...
|
||||
};
|
||||
```
|
||||
|
||||
SDS strings as already mentioned don't follow this schema, and are instead
|
||||
a single allocation with a prefix that lives *before* the address actually
|
||||
returned for the string.
|
||||
|
||||
There are advantages and disadvantages with this approach over the traditional
|
||||
approach:
|
||||
|
||||
**Disadvantage #1**: many functions return the new string as value, since sometimes SDS requires to create a new string with more space, so the most SDS API calls look like this:
|
||||
|
||||
```c
|
||||
s = sdscat(s,"Some more data");
|
||||
```
|
||||
|
||||
As you can see `s` is used as input for `sdscat` but is also set to the value
|
||||
returned by the SDS API call, since we are not sure if the call modified the
|
||||
SDS string we passed or allocated a new one. Not remembering to assign back
|
||||
the return value of `sdscat` or similar functions to the variable holding
|
||||
the SDS string will result in a bug.
|
||||
|
||||
**Disadvantage #2**: if an SDS string is shared in different places in your program you have to modify all the references when you modify the string. However most of the times when you need to share SDS strings it is much better to encapsulate them into structures with a `reference count` otherwise it is too easy to incur into memory leaks.
|
||||
|
||||
**Advantage #1**: you can pass SDS strings to functions designed for C functions without accessing a struct member or calling a function, like this:
|
||||
|
||||
```c
|
||||
printf("%s\n", sds_string);
|
||||
```
|
||||
|
||||
In most other libraries this will be something like:
|
||||
|
||||
```c
|
||||
printf("%s\n", string->buf);
|
||||
```
|
||||
|
||||
Or:
|
||||
|
||||
```c
|
||||
printf("%s\n", getStringPointer(string));
|
||||
```
|
||||
|
||||
**Advantage #2**: accessing individual chars is straightforward. C is a low level language so this is an important operation in many programs. With SDS strings accessing individual chars is very natural:
|
||||
|
||||
```c
|
||||
printf("%c %c\n", s[0], s[1]);
|
||||
```
|
||||
|
||||
With other libraries your best chance is to assign `string->buf` (or call the function to get the string pointer) to a `char` pointer and work with this. However since the other libraries may reallocate the buffer implicitly every time you call a function that may modify the string you have to get a reference to the buffer again.
|
||||
|
||||
**Advantage #3**: single allocation has better cache locality. Usually when you access a string created by a string library using a structure, you have two different allocations for the structure representing the string, and the actual buffer holding the string. Over the time the buffer is reallocated, and it is likely that it ends in a totally different part of memory compared to the structure itself. Since modern programs performances are often dominated by cache misses, SDS may perform better in many workloads.
|
||||
|
||||
SDS basics
|
||||
===
|
||||
|
||||
The type of SDS strings is just the char pointer `char *`. However SDS defines
|
||||
an `sds` type as alias of `char *` in its header file: you should use the
|
||||
`sds` type in order to make sure you remember that a given variable in your
|
||||
program holds an SDS string and not a C string, however this is not mandatory.
|
||||
|
||||
This is the simplest SDS program you can write that does something:
|
||||
|
||||
```c
|
||||
sds mystring = sdsnew("Hello World!");
|
||||
printf("%s\n", mystring);
|
||||
sdsfree(mystring);
|
||||
|
||||
output> Hello World!
|
||||
```
|
||||
|
||||
The above small program already shows a few important things about SDS:
|
||||
|
||||
* SDS strings are created, and heap allocated, via the `sdsnew()` function, or other similar functions that we'll see in a moment.
|
||||
* SDS strings can be passed to `printf()` like any other C string.
|
||||
* SDS strings require to be freed with `sdsfree()`, since they are heap allocated.
|
||||
|
||||
Creating SDS strings
|
||||
---
|
||||
|
||||
```c
|
||||
sds sdsnewlen(const void *init, size_t initlen);
|
||||
sds sdsnew(const char *init);
|
||||
sds sdsempty(void);
|
||||
sds sdsdup(const sds s);
|
||||
```
|
||||
|
||||
There are many ways to create SDS strings:
|
||||
|
||||
* The `sdsnew` function creates an SDS string starting from a C null terminated string. We already saw how it works in the above example.
|
||||
* The `sdsnewlen` function is similar to `sdsnew` but instead of creating the string assuming that the input string is null terminated, it gets an additional length parameter. This way you can create a string using binary data:
|
||||
|
||||
```c
|
||||
char buf[3];
|
||||
sds mystring;
|
||||
|
||||
buf[0] = 'A';
|
||||
buf[1] = 'B';
|
||||
buf[2] = 'C';
|
||||
mystring = sdsnewlen(buf,3);
|
||||
printf("%s of len %d\n", mystring, (int) sdslen(mystring));
|
||||
|
||||
output> ABC of len 3
|
||||
```
|
||||
|
||||
Note: `sdslen` return value is casted to `int` because it returns a `size_t`
|
||||
type. You can use the right `printf` specifier instead of casting.
|
||||
|
||||
* The `sdsempty()` function creates an empty zero-length string:
|
||||
|
||||
```c
|
||||
sds mystring = sdsempty();
|
||||
printf("%d\n", (int) sdslen(mystring));
|
||||
|
||||
output> 0
|
||||
```
|
||||
|
||||
* The `sdsdup()` function duplicates an already existing SDS string:
|
||||
|
||||
```c
|
||||
sds s1, s2;
|
||||
|
||||
s1 = sdsnew("Hello");
|
||||
s2 = sdsdup(s1);
|
||||
printf("%s %s\n", s1, s2);
|
||||
|
||||
output> Hello Hello
|
||||
```
|
||||
|
||||
Obtaining the string length
|
||||
---
|
||||
|
||||
```c
|
||||
size_t sdslen(const sds s);
|
||||
```
|
||||
|
||||
In the examples above we already used the `sdslen` function in order to get
|
||||
the length of the string. This function works like `strlen` of the libc
|
||||
except that:
|
||||
|
||||
* It runs in constant time since the length is stored in the prefix of SDS strings, so calling `sdslen` is not expensive even when called with very large strings.
|
||||
* The function is binary safe like any other SDS string function, so the length is the true length of the string regardless of the content, there is no problem if the string includes null term characters in the middle.
|
||||
|
||||
As an example of the binary safeness of SDS strings, we can run the following
|
||||
code:
|
||||
|
||||
```c
|
||||
sds s = sdsnewlen("A\0\0B",4);
|
||||
printf("%d\n", (int) sdslen(s));
|
||||
|
||||
output> 4
|
||||
```
|
||||
|
||||
Note that SDS strings are always null terminated at the end, so even in that
|
||||
case `s[4]` will be a null term, however printing the string with `printf`
|
||||
would result in just `"A"` to be printed since libc will treat the SDS string
|
||||
like a normal C string.
|
||||
|
||||
Destroying strings
|
||||
---
|
||||
|
||||
```c
|
||||
void sdsfree(sds s);
|
||||
```
|
||||
|
||||
The destroy an SDS string there is just to call `sdsfree` with the string
|
||||
pointer. Note that even empty strings created with `sdsempty` need to be
|
||||
destroyed as well otherwise they'll result into a memory leak.
|
||||
|
||||
The function `sdsfree` does not perform any operation if instead of an SDS
|
||||
string pointer, `NULL` is passed, so you don't need to check for `NULL` explicitly before calling it:
|
||||
|
||||
```c
|
||||
if (string) sdsfree(string); /* Not needed. */
|
||||
sdsfree(string); /* Same effect but simpler. */
|
||||
```
|
||||
|
||||
Concatenating strings
|
||||
---
|
||||
|
||||
Concatenating strings to other strings is likely the operation you will end
|
||||
using the most with a dynamic C string library. SDS provides different
|
||||
functions to concatenate strings to existing strings.
|
||||
|
||||
```c
|
||||
sds sdscatlen(sds s, const void *t, size_t len);
|
||||
sds sdscat(sds s, const char *t);
|
||||
```
|
||||
|
||||
The main string concatenation functions are `sdscatlen` and `sdscat` that are
|
||||
identical, the only difference being that `sdscat` does not have an explicit
|
||||
length argument since it expects a null terminated string.
|
||||
|
||||
```c
|
||||
sds s = sdsempty();
|
||||
s = sdscat(s, "Hello ");
|
||||
s = sdscat(s, "World!");
|
||||
printf("%s\n", s);
|
||||
|
||||
output> Hello World!
|
||||
```
|
||||
|
||||
Sometimes you want to cat an SDS string to another SDS string, so you don't
|
||||
need to specify the length, but at the same time the string does not need to
|
||||
be null terminated but can contain any binary data. For this there is a
|
||||
special function:
|
||||
|
||||
```c
|
||||
sds sdscatsds(sds s, const sds t);
|
||||
```
|
||||
|
||||
Usage is straightforward:
|
||||
|
||||
```c
|
||||
sds s1 = sdsnew("aaa");
|
||||
sds s2 = sdsnew("bbb");
|
||||
s1 = sdscatsds(s1,s2);
|
||||
sdsfree(s2);
|
||||
printf("%s\n", s1);
|
||||
|
||||
output> aaabbb
|
||||
```
|
||||
|
||||
Sometimes you don't want to append any special data to the string, but you want
|
||||
to make sure that there are at least a given number of bytes composing the
|
||||
whole string.
|
||||
|
||||
```c
|
||||
sds sdsgrowzero(sds s, size_t len);
|
||||
```
|
||||
|
||||
The `sdsgrowzero` function will do nothing if the current string length is
|
||||
already `len` bytes, otherwise it will enlarge the string to `len` just padding
|
||||
it with zero bytes.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello");
|
||||
s = sdsgrowzero(s,6);
|
||||
s[5] = '!'; /* We are sure this is safe because of sdsgrowzero() */
|
||||
printf("%s\n', s);
|
||||
|
||||
output> Hello!
|
||||
```
|
||||
|
||||
Formatting strings
|
||||
---
|
||||
|
||||
There is a special string concatenation function that accepts a `printf` alike
|
||||
format specifier and cats the formatted string to the specified string.
|
||||
|
||||
```c
|
||||
sds sdscatprintf(sds s, const char *fmt, ...) {
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```c
|
||||
sds s;
|
||||
int a = 10, b = 20;
|
||||
s = sdsnew("The sum is: ");
|
||||
s = sdscatprintf(s,"%d+%d = %d",a,b,a+b);
|
||||
```
|
||||
|
||||
Often you need to create SDS string directly from `printf` format specifiers.
|
||||
Because `sdscatprintf` is actually a function that concatenates strings, all
|
||||
you need is to concatenate your string to an empty string:
|
||||
|
||||
|
||||
```c
|
||||
char *name = "Anna";
|
||||
int loc = 2500;
|
||||
sds s;
|
||||
s = sdscatprintf(sdsempty(), "%s wrote %d lines of LISP\n", name, loc);
|
||||
```
|
||||
|
||||
You can use `sdscatprintf` in order to convert numbers into SDS strings:
|
||||
|
||||
```c
|
||||
int some_integer = 100;
|
||||
sds num = sdscatprintf(sdsempty(),"%d\n", some_integer);
|
||||
```
|
||||
|
||||
However this is slow and we have a special function to make it efficient.
|
||||
|
||||
Fast number to string operations
|
||||
---
|
||||
|
||||
Creating an SDS string from an integer may be a common operation in certain
|
||||
kind of programs, and while you may do this with `sdscatprintf` the performance
|
||||
hit is big, so SDS provides a specialized function.
|
||||
|
||||
```c
|
||||
sds sdsfromlonglong(long long value);
|
||||
```
|
||||
|
||||
Use it like this:
|
||||
|
||||
```c
|
||||
sds s = sdsfromlonglong(10000);
|
||||
printf("%d\n", (int) sdslen(s));
|
||||
|
||||
output> 5
|
||||
```
|
||||
|
||||
Trimming strings and getting ranges
|
||||
---
|
||||
|
||||
String trimming is a common operation where a set of characters are
|
||||
removed from the left and the right of the string. Another useful operation
|
||||
regarding strings is the ability to just take a range out of a larger
|
||||
string.
|
||||
|
||||
```c
|
||||
void sdstrim(sds s, const char *cset);
|
||||
void sdsrange(sds s, int start, int end);
|
||||
```
|
||||
|
||||
SDS provides both the operations with the `sdstrim` and `sdsrange` functions.
|
||||
However note that both functions work differently than most functions modifying
|
||||
SDS strings since the return value is void: basically those functions always
|
||||
destructively modify the passed SDS string, never allocating a new one, because
|
||||
both trimming and ranges will never need more room: the operations can only
|
||||
remove characters from the original string.
|
||||
|
||||
Because of this behavior, both functions are fast and don't involve reallocation.
|
||||
|
||||
This is an example of string trimming where newlines and spaces are removed
|
||||
from an SDS strings:
|
||||
|
||||
```c
|
||||
sds s = sdsnew(" my string\n\n ");
|
||||
sdstrim(s," \n");
|
||||
printf("-%s-\n",s);
|
||||
|
||||
output> -my string-
|
||||
```
|
||||
|
||||
Basically `sdstrim` takes the SDS string to trim as first argument, and a
|
||||
null terminated set of characters to remove from left and right of the string.
|
||||
The characters are removed as long as they are not interrupted by a character
|
||||
that is not in the list of characters to trim: this is why the space between
|
||||
`"my"` and `"string"` was preserved in the above example.
|
||||
|
||||
Taking ranges is similar, but instead to take a set of characters, it takes
|
||||
to indexes, representing the start and the end as specified by zero-based
|
||||
indexes inside the string, to obtain the range that will be retained.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello World!");
|
||||
sdsrange(s,1,4);
|
||||
printf("-%s-\n");
|
||||
|
||||
output> -ello-
|
||||
```
|
||||
|
||||
Indexes can be negative to specify a position starting from the end of the
|
||||
string, so that `-1` means the last character, `-2` the penultimate, and so forth:
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello World!");
|
||||
sdsrange(s,6,-1);
|
||||
printf("-%s-\n");
|
||||
sdsrange(s,0,-2);
|
||||
printf("-%s-\n");
|
||||
|
||||
output> -World!-
|
||||
output> -World-
|
||||
```
|
||||
|
||||
`sdsrange` is very useful when implementing networking servers processing
|
||||
a protocol or sending messages. For example the following code is used
|
||||
implementing the write handler of the Redis Cluster message bus between
|
||||
nodes:
|
||||
|
||||
```c
|
||||
void clusterWriteHandler(..., int fd, void *privdata, ...) {
|
||||
clusterLink *link = (clusterLink*) privdata;
|
||||
ssize_t nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf));
|
||||
if (nwritten <= 0) {
|
||||
/* Error handling... */
|
||||
}
|
||||
sdsrange(link->sndbuf,nwritten,-1);
|
||||
... more code here ...
|
||||
}
|
||||
```
|
||||
|
||||
Every time the socket of the node we want to send the message to is writable
|
||||
we attempt to write as much bytes as possible, and we use `sdsrange` in order
|
||||
to remove from the buffer what was already sent.
|
||||
|
||||
The function to queue new messages to send to some node in the cluster will
|
||||
simply use `sdscatlen` in order to put more data in the send buffer.
|
||||
|
||||
Note that the Redis Cluster bus implements a binary protocol, but since SDS
|
||||
is binary safe this is not a problem, so the goal of SDS is not just to provide
|
||||
an high level string API for the C programmer but also dynamically allocated
|
||||
buffers that are easy to manage.
|
||||
|
||||
String copying
|
||||
---
|
||||
|
||||
The most dangerous and infamus function of the standard C library is probably
|
||||
`strcpy`, so perhaps it is funny how in the context of better designed dynamic
|
||||
string libraries the concept of copying strings is almost irrelevant. Usually
|
||||
what you do is to create strings with the content you want, or concatenating
|
||||
more content as needed.
|
||||
|
||||
However SDS features a string copy function that is useful in performance
|
||||
critical code sections, however I guess its practical usefulness is limited
|
||||
as the function never managed to get called in the context of the 50k
|
||||
lines of code composing the Redis code base.
|
||||
|
||||
```c
|
||||
sds sdscpylen(sds s, const char *t, size_t len);
|
||||
sds sdscpy(sds s, const char *t);
|
||||
```
|
||||
|
||||
The string copy function of SDS is called `sdscpylen` and works like that:
|
||||
|
||||
```c
|
||||
s = sdsnew("Hello World!");
|
||||
s = sdscpylen(s,"Hello Superman!",15);
|
||||
```
|
||||
|
||||
As you can see the function receives as input the SDS string `s`, but also
|
||||
returns an SDS string. This is common to many SDS functions that modify the
|
||||
string: this way the returned SDS string may be the original one modified
|
||||
or a newly allocated one (for example if there was not enough room in the
|
||||
old SDS string).
|
||||
|
||||
The `sdscpylen` will simply replace what was in the old SDS string with the
|
||||
new data you pass using the pointer and length argument. There is a similar
|
||||
function called `sdscpy` that does not need a length but expects a null
|
||||
terminated string instead.
|
||||
|
||||
You may wonder why it makes sense to have a string copy function in the
|
||||
SDS library, since you can simply create a new SDS string from scratch
|
||||
with the new value instead of copying the value in an existing SDS string.
|
||||
The reason is efficiency: `sdsnewlen` will always allocate a new string
|
||||
while `sdscpylen` will try to reuse the existing string if there is enough
|
||||
room to old the new content specified by the user, and will allocate a new
|
||||
one only if needed.
|
||||
|
||||
Quoting strings
|
||||
---
|
||||
|
||||
In order to provide consistent output to the program user, or for debugging
|
||||
purposes, it is often important to turn a string that may contain binary
|
||||
data or special characters into a quoted string. Here for quoted string
|
||||
we mean the common format for String literals in programming source code.
|
||||
However today this format is also part of the well known serialization formats
|
||||
like JSON and CSV, so it definitely escaped the simple goal of representing
|
||||
literals strings in the source code of programs.
|
||||
|
||||
An example of quoted string literal is the following:
|
||||
|
||||
```c
|
||||
"\x00Hello World\n"
|
||||
```
|
||||
|
||||
The first byte is a zero byte while the last byte is a newline, so there are
|
||||
two non alphanumerical characters inside the string.
|
||||
|
||||
SDS uses a concatenation function for this goal, that concatenates to an
|
||||
existing string the quoted string representation of the input string.
|
||||
|
||||
```c
|
||||
sds sdscatrepr(sds s, const char *p, size_t len);
|
||||
```
|
||||
|
||||
The `scscatrepr` (where `repr` means *representation*) follows the usualy
|
||||
SDS string function rules accepting a char pointer and a length, so you can
|
||||
use it with SDS strings, normal C strings by using strlen() as `len` argument,
|
||||
or binary data. The following is an example usage:
|
||||
|
||||
```c
|
||||
sds s1 = sdsnew("abcd");
|
||||
sds s2 = sdsempty();
|
||||
s[1] = 1;
|
||||
s[2] = 2;
|
||||
s[3] = '\n';
|
||||
s2 = sdscatrepr(s2,s1,sdslen(s1));
|
||||
printf("%s\n", s2);
|
||||
|
||||
output> "a\x01\x02\n"
|
||||
```
|
||||
|
||||
This is the rules `sdscatrepr` uses for conversion:
|
||||
|
||||
* `\` and `"` are quoted with a backslash.
|
||||
* It quotes special characters `'\n'`, `'\r'`, `'\t'`, `'\a'` and `'\b'`.
|
||||
* All the other non printable characters not passing the `isprint` test are quoted in `\x..` form, that is: backslash followed by `x` followed by two digit hex number representing the character byte value.
|
||||
* The function always adds initial and final double quotes characters.
|
||||
|
||||
There is an SDS function that is able to perform the reverse conversion and is
|
||||
documented in the *Tokenization* section below.
|
||||
|
||||
Tokenization
|
||||
---
|
||||
|
||||
Tokenization is the process of splitting a larger string into smaller strings.
|
||||
In this specific case, the split is performed specifying another string that
|
||||
acts as separator. For example in the following string there are two substrings
|
||||
that are separated by the `|-|` separator:
|
||||
|
||||
```
|
||||
foo|-|bar|-|zap
|
||||
```
|
||||
|
||||
A more common separator that consists of a single character is the comma:
|
||||
|
||||
```
|
||||
foo,bar,zap
|
||||
```
|
||||
|
||||
In many progrems it is useful to process a line in order to obtain the sub
|
||||
strings it is composed of, so SDS provides a function that returns an
|
||||
array of SDS strings given a string and a separator.
|
||||
|
||||
```c
|
||||
sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
|
||||
void sdsfreesplitres(sds *tokens, int count);
|
||||
```
|
||||
|
||||
As usually the function can work with both SDS strings or normal C strings.
|
||||
The first two arguments `s` and `len` specify the string to tokenize, and the
|
||||
other two arguments `sep` and `seplen` the separator to use during the
|
||||
tokenization. The final argument `count` is a pointer to an integer that will
|
||||
be set to the number of tokens (sub strings) returned.
|
||||
|
||||
The return value is a heap allocated array of SDS strings.
|
||||
|
||||
```c
|
||||
sds *tokens;
|
||||
int count, j;
|
||||
|
||||
sds line = sdsnew("Hello World!");
|
||||
tokens = sdssplitlen(line,sdslen(line)," ",1,&count);
|
||||
|
||||
for (j = 0; j < count; j++)
|
||||
printf("%s\n", tokens[j]);
|
||||
sdsfreesplitres(tokens,count);
|
||||
|
||||
output> Hello
|
||||
output> World!
|
||||
```
|
||||
|
||||
The returned array is heap allocated, and the single elements of the array
|
||||
are normal SDS strings. You can free everything calling `sdsfreesplitres`
|
||||
as in the example. Alternativey you are free to release the array yourself
|
||||
using the `free` function and use and/or free the individual SDS strings
|
||||
as usually.
|
||||
|
||||
A valid approach is to set the array elements you reused in some way to
|
||||
`NULL`, and use `sdsfreesplitres` to free all the rest.
|
||||
|
||||
Command line oriented tokenization
|
||||
---
|
||||
|
||||
Splitting by a separator is a useful operation, but usually it is not enough
|
||||
to perform one of the most common tasks involving some non trivial string
|
||||
manipulation, that is, implementing a **Command Line Interface** for a program.
|
||||
|
||||
This is why SDS also provides an additional function that allows you to split
|
||||
arguments provided by the user via the keyboard in an interactive manner, or
|
||||
via a file, network, or any other mean, into tokens.
|
||||
|
||||
```c
|
||||
sds *sdssplitargs(const char *line, int *argc);
|
||||
```
|
||||
|
||||
The `sdssplitargs` function returns an array of SDS strings exactly like
|
||||
`sdssplitlen`. The function to free the result is also identical, and is
|
||||
`sdsfreesplitres`. The difference is in the way the tokenization is performed.
|
||||
|
||||
For example if the input is the following line:
|
||||
|
||||
```
|
||||
call "Sabrina" and "Mark Smith\n"
|
||||
```
|
||||
|
||||
The function will return the following tokens:
|
||||
|
||||
* "call"
|
||||
* "Sabrina"
|
||||
* "and"
|
||||
* "Mark Smith\n"
|
||||
|
||||
Basically different tokens need to be separated by one or more spaces, and
|
||||
every single token can also be a quoted string in the same format that
|
||||
`sdscatrepr` is able to emit.
|
||||
|
||||
String joining
|
||||
---
|
||||
|
||||
There are two functions doing the reverse of tokenization by joining strings
|
||||
into a single one.
|
||||
|
||||
```c
|
||||
sds sdsjoin(char **argv, int argc, char *sep, size_t seplen);
|
||||
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
|
||||
```
|
||||
|
||||
The two functions take as input an array of strings of length `argc` and
|
||||
a separator and its length, and produce as output an SDS string consisting
|
||||
of all the specified strings separated by the specified separator.
|
||||
|
||||
The difference between `sdsjoin` and `sdsjoinsds` is that the former accept
|
||||
C null terminated strings as input while the latter requires all the strings
|
||||
in the array to be SDS strings. However because of this only `sdsjoinsds` is
|
||||
able to deal with binary data.
|
||||
|
||||
```c
|
||||
char *tokens[3] = {"foo","bar","zap"};
|
||||
sds s = sdsjoin(tokens,3,"|",1);
|
||||
printf("%s\n", s);
|
||||
|
||||
output> foo|bar|zap
|
||||
```
|
||||
|
||||
Error handling
|
||||
---
|
||||
|
||||
All the SDS functions that return an SDS pointer may also return `NULL` on
|
||||
out of memory, this is basically the only check you need to perform.
|
||||
|
||||
However many modern C programs handle out of memory simply aborting the program
|
||||
so you may want to do this as well by wrapping `malloc` and other related
|
||||
memory allocation calls directly.
|
||||
|
||||
SDS internals and advanced usage
|
||||
===
|
||||
|
||||
At the very beginning of this documentation it was explained how SDS strings
|
||||
are allocated, however the prefix stored before the pointer returned to the
|
||||
user was classified as an *header* without further details. For an advanced
|
||||
usage it is better to dig more into the internals of SDS and show the
|
||||
structure implementing it:
|
||||
|
||||
```c
|
||||
struct sdshdr {
|
||||
int len;
|
||||
int free;
|
||||
char buf[];
|
||||
};
|
||||
```
|
||||
|
||||
As you can see, the structure may resemble the one of a conventional string
|
||||
library, however the `buf` field of the structure is different since it is
|
||||
not a pointer but an array without any length declared, so `buf` actually
|
||||
points at the first byte just after the `free` integer. So in order to create
|
||||
an SDS string we just allocate a piece of memory that is as large as the
|
||||
`sdshdr` structure plus the length of our string, plus an additional byte
|
||||
for the mandatory null term that every SDS string has.
|
||||
|
||||
The `len` field of the structure is quite obvious, and is the current length
|
||||
of the SDS string, always computed every time the string is modified via
|
||||
SDS function calls. The `free` field instead represents the amount of free
|
||||
memory in the current allocation that can be used to store more characters.
|
||||
|
||||
So the actual SDS layout is this one:
|
||||
|
||||
+------------+------------------------+-----------+---------------\
|
||||
| Len | Free | H E L L O W O R L D \n | Null term | Free space \
|
||||
+------------+------------------------+-----------+---------------\
|
||||
|
|
||||
`-> Pointer returned to the user.
|
||||
|
||||
You may wonder why there is some free space at the end of the string, it
|
||||
looks like a waste. Actually after a new SDS string is created, there is no
|
||||
free space at the end at all: the allocation will be as small as possible to
|
||||
just hold the header, string, and null term. However other access patterns
|
||||
will create extra free space at the end, like in the following program:
|
||||
|
||||
```c
|
||||
s = sdsempty();
|
||||
s = sdscat(s,"foo");
|
||||
s = sdscat(s,"bar");
|
||||
s = sdscat(s,"123");
|
||||
```
|
||||
|
||||
Since SDS tries to be efficient it can't afford to reallocate the string every
|
||||
time new data is appended, since this would be very inefficient, so it uses
|
||||
the **preallocation of some free space** every time you enlarge the string.
|
||||
|
||||
The preallocation algorithm used is the following: every time the string
|
||||
is reallocated in order to hold more bytes, the actual allocation size performed
|
||||
is two times the minimum required. So for instance if the string currently
|
||||
is holding 30 bytes, and we concatenate 2 more bytes, instead of allocating 32
|
||||
bytes in total SDS will allocate 64 bytes.
|
||||
|
||||
However there is an hard limit to the allocation it can perform ahead, and is
|
||||
defined by `SDS_MAX_PREALLOC`. SDS will never allocate more than 1MB of
|
||||
additional space (by default, you can change this default).
|
||||
|
||||
Shrinking strings
|
||||
---
|
||||
|
||||
```c
|
||||
sds sdsRemoveFreeSpace(sds s);
|
||||
size_t sdsAllocSize(sds s);
|
||||
```
|
||||
|
||||
Sometimes there are class of programs that require to use very little memory.
|
||||
After strings concatenations, trimming, ranges, the string may end having
|
||||
a non trivial amount of additional space at the end.
|
||||
|
||||
It is possible to resize a string back to its minimal size in order to hold
|
||||
the current content by using the function `sdsRemoveFreeSpace`.
|
||||
|
||||
```c
|
||||
s = sdsRemoveFreeSpace(s);
|
||||
```
|
||||
|
||||
There is also a function that can be used in order to get the size of the
|
||||
total allocation for a given string, and is called `sdsAllocSize`.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Ladies and gentlemen");
|
||||
s = sdscat(s,"... welcome to the C language.");
|
||||
printf("%d\n", (int) sdsAllocSize(s));
|
||||
s = sdsRemoveFreeSpace(s);
|
||||
printf("%d\n", (int) sdsAllocSize(s));
|
||||
|
||||
output> 109
|
||||
output> 59
|
||||
```
|
||||
|
||||
NOTE: SDS Low level API use cammelCase in order to warn you that you are playing with the fire.
|
||||
|
||||
Manual modifications of SDS strings
|
||||
---
|
||||
|
||||
void sdsupdatelen(sds s);
|
||||
|
||||
Sometimes you may want to hack with an SDS string manually, without using
|
||||
SDS functions. In the following example we implicitly change the length
|
||||
of the string, however we want the logical length to reflect the null terminated
|
||||
C string.
|
||||
|
||||
The function `sdsupdatelen` does just that, updating the internal length
|
||||
information for the specified string to the length obtained via `strlen`.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("foobar");
|
||||
s[2] = '\0';
|
||||
printf("%d\n", sdslen(s));
|
||||
sdsupdatelen(s);
|
||||
printf("%d\n", sdslen(s));
|
||||
|
||||
output> 6
|
||||
output> 2
|
||||
```
|
||||
|
||||
Sharing SDS strings
|
||||
---
|
||||
|
||||
If you are writing a program in which it is advantageous to share the same
|
||||
SDS string across different data structures, it is absolutely advised to
|
||||
encapsulate SDS strings into structures that remember the number of references
|
||||
of the string, with functions to increment and decrement the number of references.
|
||||
|
||||
This approach is a memory management technique called *reference counting* and
|
||||
in the context of SDS has two advantages:
|
||||
|
||||
* It is less likely that you'll create memory leaks or bugs due to non freeing SDS strings or freeing already freed strings.
|
||||
* You'll not need to update every reference to an SDS string when you modify it (since the new SDS string may point to a different memory location).
|
||||
|
||||
While this is definitely a very common programming technique I'll outline
|
||||
the basic ideas here. You create a structure like that:
|
||||
|
||||
```c
|
||||
struct mySharedString {
|
||||
int refcount;
|
||||
sds string;
|
||||
}
|
||||
```
|
||||
|
||||
When new strings are created, the structure is allocated and returned with
|
||||
`refcount` set to 1. The you have two functions to change the reference count
|
||||
of the shared string:
|
||||
|
||||
* `incrementStringRefCount` will simply increment `refcount` of 1 in the structure. It will be called every time you add a reference to the string on some new data structure, variable, or whatever.
|
||||
* `decrementStringRefCount` is used when you remove a reference. This function is however special since when the `refcount` drops to zero, it automatically frees the SDS string, and the `mySharedString` structure as well.
|
||||
|
||||
Interactions with heap checkers
|
||||
---
|
||||
|
||||
Because SDS returns pointers into the middle of memory chunks allocated with
|
||||
`malloc`, heap checkers may have issues, however:
|
||||
|
||||
* The popular Valgrind program will detect SDS strings are *possibly lost* memory and never as *definitely lost*, so it is easy to tell if there is a leak or not. I used Valgrind with Redis for years and every real leak was consistently detected as "definitely lost".
|
||||
* OSX instrumentation tools don't detect SDS strings as leaks but are able to correctly handle pointers pointing to the middle of memory chunks.
|
||||
|
||||
Zero copy append from syscalls
|
||||
----
|
||||
|
||||
At this point you should have all the tools to dig more inside the SDS
|
||||
library by reading the source code, however there is an interesting pattern
|
||||
you can mount using the low level API exported, that is used inside Redis
|
||||
in order to improve performances of the networking code.
|
||||
|
||||
Using `sdsIncrLen()` and `sdsMakeRoomFor()` it is possible to mount the
|
||||
following schema, to cat bytes coming from the kernel to the end of an
|
||||
sds string without copying into an intermediate buffer:
|
||||
|
||||
```c
|
||||
oldlen = sdslen(s);
|
||||
s = sdsMakeRoomFor(s, BUFFER_SIZE);
|
||||
nread = read(fd, s+oldlen, BUFFER_SIZE);
|
||||
... check for nread <= 0 and handle it ...
|
||||
sdsIncrLen(s, nread);
|
||||
```
|
||||
|
||||
`sdsIncrLen` is documented inside the source code of `sds.c`.
|
||||
|
||||
Embedding SDS into your project
|
||||
===
|
||||
|
||||
This is as simple as copying the following files inside your
|
||||
project:
|
||||
|
||||
* sds.c
|
||||
* sds.h
|
||||
* sdsalloc.h
|
||||
|
||||
The source code is small and every C99 compiler should deal with
|
||||
it without issues.
|
||||
|
||||
Using a different allocator for SDS
|
||||
===
|
||||
|
||||
Internally sds.c uses the allocator defined into `sdsalloc.h`. This header
|
||||
file just defines macros for malloc, realloc and free, and by default libc
|
||||
`malloc()`, `realloc()` and `free()` are used. Just edit this file in order
|
||||
to change the name of the allocation functions.
|
||||
|
||||
The program using SDS can call the SDS allocator in order to manipulate
|
||||
SDS pointers (usually not needed but sometimes the program may want to
|
||||
do advanced things) by using the API exported by SDS in order to call the
|
||||
allocator used. This is especially useful when the program linked to SDS
|
||||
is using a different allocator compared to what SDS is using.
|
||||
|
||||
The API to access the allocator used by SDS is composed of three functions: `sds_malloc()`, `sds_realloc()` and `sds_free()`.
|
||||
|
||||
Credits and license
|
||||
===
|
||||
|
||||
SDS was created by Salvatore Sanfilippo and is released under the BDS two clause license. See the LICENSE file in this source distribution for more information.
|
||||
|
||||
Oran Agra improved SDS version 2 by adding dynamic sized headers in order to
|
||||
save memory for small strings and allow strings greater than 4GB.
|
||||
1300
deps/sds/sds.c
vendored
1300
deps/sds/sds.c
vendored
File diff suppressed because it is too large
Load Diff
283
deps/sds/sds.h
vendored
283
deps/sds/sds.h
vendored
@@ -1,283 +0,0 @@
|
||||
/* SDSLib 2.0 -- A C dynamic strings library
|
||||
*
|
||||
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* Copyright (c) 2015, Oran Agra
|
||||
* Copyright (c) 2015, Redis Labs, Inc
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __SDS_H
|
||||
#define __SDS_H
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#define SDS_MAX_PREALLOC (1024*1024)
|
||||
extern const char *SDS_NOINIT;
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef char *sds;
|
||||
|
||||
/* Note: sdshdr5 is never used, we just access the flags byte directly.
|
||||
* However is here to document the layout of type 5 SDS strings. */
|
||||
struct __attribute__ ((__packed__)) sdshdr5 {
|
||||
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr8 {
|
||||
uint8_t len; /* used */
|
||||
uint8_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr16 {
|
||||
uint16_t len; /* used */
|
||||
uint16_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr32 {
|
||||
uint32_t len; /* used */
|
||||
uint32_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr64 {
|
||||
uint64_t len; /* used */
|
||||
uint64_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
|
||||
#define SDS_TYPE_5 0
|
||||
#define SDS_TYPE_8 1
|
||||
#define SDS_TYPE_16 2
|
||||
#define SDS_TYPE_32 3
|
||||
#define SDS_TYPE_64 4
|
||||
#define SDS_TYPE_MASK 7
|
||||
#define SDS_TYPE_BITS 3
|
||||
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
|
||||
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
|
||||
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
|
||||
|
||||
static inline size_t sdslen(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
return SDS_TYPE_5_LEN(flags);
|
||||
case SDS_TYPE_8:
|
||||
return SDS_HDR(8,s)->len;
|
||||
case SDS_TYPE_16:
|
||||
return SDS_HDR(16,s)->len;
|
||||
case SDS_TYPE_32:
|
||||
return SDS_HDR(32,s)->len;
|
||||
case SDS_TYPE_64:
|
||||
return SDS_HDR(64,s)->len;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline size_t sdsavail(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5: {
|
||||
return 0;
|
||||
}
|
||||
case SDS_TYPE_8: {
|
||||
SDS_HDR_VAR(8,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_16: {
|
||||
SDS_HDR_VAR(16,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_32: {
|
||||
SDS_HDR_VAR(32,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_64: {
|
||||
SDS_HDR_VAR(64,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void sdssetlen(sds s, size_t newlen) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
{
|
||||
unsigned char *fp = ((unsigned char*)s)-1;
|
||||
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
|
||||
}
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->len = newlen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void sdsinclen(sds s, size_t inc) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
{
|
||||
unsigned char *fp = ((unsigned char*)s)-1;
|
||||
unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc;
|
||||
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
|
||||
}
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->len += inc;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* sdsalloc() = sdsavail() + sdslen() */
|
||||
static inline size_t sdsalloc(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
return SDS_TYPE_5_LEN(flags);
|
||||
case SDS_TYPE_8:
|
||||
return SDS_HDR(8,s)->alloc;
|
||||
case SDS_TYPE_16:
|
||||
return SDS_HDR(16,s)->alloc;
|
||||
case SDS_TYPE_32:
|
||||
return SDS_HDR(32,s)->alloc;
|
||||
case SDS_TYPE_64:
|
||||
return SDS_HDR(64,s)->alloc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void sdssetalloc(sds s, size_t newlen) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
/* Nothing to do, this type has no total allocation info. */
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->alloc = newlen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sds sdsnewlen(const void *init, size_t initlen);
|
||||
sds sdsnew(const char *init);
|
||||
sds sdsempty(void);
|
||||
sds sdsdup(const sds s);
|
||||
void sdsfree(sds s);
|
||||
sds sdsgrowzero(sds s, size_t len);
|
||||
sds sdscatlen(sds s, const void *t, size_t len);
|
||||
sds sdscat(sds s, const char *t);
|
||||
sds sdscatsds(sds s, const sds t);
|
||||
sds sdscpylen(sds s, const char *t, size_t len);
|
||||
sds sdscpy(sds s, const char *t);
|
||||
|
||||
sds sdscatvprintf(sds s, const char *fmt, va_list ap);
|
||||
#ifdef __GNUC__
|
||||
sds sdscatprintf(sds s, const char *fmt, ...)
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
#else
|
||||
sds sdscatprintf(sds s, const char *fmt, ...);
|
||||
#endif
|
||||
|
||||
sds sdscatfmt(sds s, char const *fmt, ...);
|
||||
sds sdstrim(sds s, const char *cset);
|
||||
void sdsrange(sds s, ssize_t start, ssize_t end);
|
||||
void sdsupdatelen(sds s);
|
||||
void sdsclear(sds s);
|
||||
int sdscmp(const sds s1, const sds s2);
|
||||
sds *sdssplitlen(const char *s, ssize_t len, const char *sep, int seplen, int *count);
|
||||
void sdsfreesplitres(sds *tokens, int count);
|
||||
void sdstolower(sds s);
|
||||
void sdstoupper(sds s);
|
||||
sds sdsfromlonglong(long long value);
|
||||
sds sdscatrepr(sds s, const char *p, size_t len);
|
||||
sds *sdssplitargs(const char *line, int *argc);
|
||||
sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
|
||||
sds sdsjoin(char **argv, int argc, char *sep);
|
||||
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
|
||||
|
||||
/* Low level functions exposed to the user API */
|
||||
sds sdsMakeRoomFor(sds s, size_t addlen);
|
||||
void sdsIncrLen(sds s, ssize_t incr);
|
||||
sds sdsRemoveFreeSpace(sds s);
|
||||
size_t sdsAllocSize(sds s);
|
||||
void *sdsAllocPtr(sds s);
|
||||
|
||||
/* Export the allocator used by SDS to the program using SDS.
|
||||
* Sometimes the program SDS is linked to, may use a different set of
|
||||
* allocators, but may want to allocate or free things that SDS will
|
||||
* respectively free or allocate. */
|
||||
void *sds_malloc(size_t size);
|
||||
void *sds_realloc(void *ptr, size_t size);
|
||||
void sds_free(void *ptr);
|
||||
|
||||
#ifdef REDIS_TEST
|
||||
int sdsTest(int argc, char *argv[]);
|
||||
#endif
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
42
deps/sds/sdsalloc.h
vendored
42
deps/sds/sdsalloc.h
vendored
@@ -1,42 +0,0 @@
|
||||
/* SDSLib 2.0 -- A C dynamic strings library
|
||||
*
|
||||
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* Copyright (c) 2015, Oran Agra
|
||||
* Copyright (c) 2015, Redis Labs, Inc
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/* SDS allocator selection.
|
||||
*
|
||||
* This file is used in order to change the SDS allocator at compile time.
|
||||
* Just define the following defines to what you want to use. Also add
|
||||
* the include of your alternate allocator if needed (not needed in order
|
||||
* to use the default libc allocator). */
|
||||
|
||||
#define s_malloc malloc
|
||||
#define s_realloc realloc
|
||||
#define s_free free
|
||||
57
deps/sds/testhelp.h
vendored
57
deps/sds/testhelp.h
vendored
@@ -1,57 +0,0 @@
|
||||
/* This is a really minimal testing framework for C.
|
||||
*
|
||||
* Example:
|
||||
*
|
||||
* test_cond("Check if 1 == 1", 1==1)
|
||||
* test_cond("Check if 5 > 10", 5 > 10)
|
||||
* test_report()
|
||||
*
|
||||
* ----------------------------------------------------------------------------
|
||||
*
|
||||
* Copyright (c) 2010-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __TESTHELP_H
|
||||
#define __TESTHELP_H
|
||||
|
||||
int __failed_tests = 0;
|
||||
int __test_num = 0;
|
||||
#define test_cond(descr,_c) do { \
|
||||
__test_num++; printf("%d - %s: ", __test_num, descr); \
|
||||
if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \
|
||||
} while(0);
|
||||
#define test_report() do { \
|
||||
printf("%d tests, %d passed, %d failed\n", __test_num, \
|
||||
__test_num-__failed_tests, __failed_tests); \
|
||||
if (__failed_tests) { \
|
||||
printf("=== WARNING === We have failed tests here...\n"); \
|
||||
exit(1); \
|
||||
} \
|
||||
} while(0);
|
||||
|
||||
#endif
|
||||
@@ -16,6 +16,11 @@
|
||||
#ifndef _MAAT_H_
|
||||
#define _MAAT_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <netinet/in.h>
|
||||
|
||||
@@ -31,30 +36,66 @@ enum ip_type {
|
||||
IP_TYPE_V6
|
||||
};
|
||||
|
||||
struct ip_data {
|
||||
typedef struct ip_data_s {
|
||||
enum ip_type type;
|
||||
union {
|
||||
uint32_t ipv4;
|
||||
uint32_t ipv6[4];
|
||||
};
|
||||
};
|
||||
} ip_data_t;
|
||||
|
||||
#define MAAT_RULE_UPDATE_TYPE_FULL 1
|
||||
#define MAAT_RULE_UPDATE_TYPE_INC 2
|
||||
|
||||
typedef void maat_start_callback_t(int update_type, void *u_param);
|
||||
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
|
||||
typedef void maat_finish_callback_t(void *u_para);
|
||||
|
||||
typedef void maat_plugin_ex_new_func_t(int table_id, const char *key, const char *table_line, void **ad, long argl, void *argp);
|
||||
typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void *argp);
|
||||
typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp);
|
||||
|
||||
/* maat_instance options API */
|
||||
struct maat_options;
|
||||
struct maat_options* maat_options_new(void);
|
||||
int maat_options_set_worker_thread_number(struct maat_options *opts, size_t nr_worker_threads);
|
||||
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
|
||||
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
|
||||
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
|
||||
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len);
|
||||
int maat_options_set_deferred_load_on(struct maat_options *opts);
|
||||
int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir);
|
||||
int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir);
|
||||
|
||||
struct maat *maat_new(struct maat_options options, const char* table_info_path);
|
||||
/* maat_instance API */
|
||||
struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
|
||||
void maat_free(struct maat *instance);
|
||||
|
||||
/* maat table API */
|
||||
int maat_table_get_id(struct maat *instance, const char *table_name);
|
||||
|
||||
int maat_table_callback_register(struct maat *instance, int table_id,
|
||||
maat_start_callback_t *start,
|
||||
maat_update_callback_t *update,
|
||||
maat_finish_callback_t *finish,
|
||||
void *u_para);
|
||||
/* maat plugin table API */
|
||||
int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
|
||||
maat_plugin_ex_new_func_t *new_func,
|
||||
maat_plugin_ex_free_func_t *free_func,
|
||||
maat_plugin_ex_dup_func_t *dup_func,
|
||||
long argl, void *argp);
|
||||
|
||||
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
|
||||
const char *key, size_t key_len);
|
||||
/* maat scan API */
|
||||
struct maat_state;
|
||||
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
|
||||
unsigned int intval, int results[], size_t *n_result,
|
||||
struct maat_state *state);
|
||||
|
||||
int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
|
||||
const struct ip_data *ip, int results[], size_t *n_result,
|
||||
const ip_data_t *ip, int results[], size_t *n_result,
|
||||
struct maat_state *state);
|
||||
|
||||
int maat_scan_string(struct maat *instance, int table_id, int thread_id,
|
||||
@@ -71,5 +112,8 @@ void maat_scan_stream_close(struct maat_stream **stream);
|
||||
|
||||
void maat_state_reset(struct maat_state *state);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
27
include/utils.h
Normal file
27
include/utils.h
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_utils.h
|
||||
* Description: maat utils entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _UTILS_H_
|
||||
#define _UTILS_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
10
scanner/CMakeLists.txt
Normal file
10
scanner/CMakeLists.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
add_definitions(-D_GNU_SOURCE)
|
||||
add_definitions(-fPIC)
|
||||
|
||||
include_directories(${PROJECT_SOURCE_DIR}/deps)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/include)
|
||||
|
||||
add_library(adapter-static adapter_hs.cpp bool_matcher.cpp)
|
||||
|
||||
#add_executable(adapter_gtest adapter_hs_gtest.cpp)
|
||||
target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static ipmatcher-static)
|
||||
117
scanner/IPMatcher.h
Normal file
117
scanner/IPMatcher.h
Normal file
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
*
|
||||
* Copyright (c) 2020
|
||||
* String Algorithms Research Group
|
||||
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
|
||||
* National Engineering Laboratory for Information Security Technologies (NELIST)
|
||||
* All rights reserved
|
||||
*
|
||||
* Written by: LU YUHAI (luyuhai@iie.ac.cn)
|
||||
* Last modification: 2020-04-20
|
||||
*
|
||||
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
|
||||
* Usage for direct or indirect commercial advantage is not allowed without
|
||||
* written permission from the authors.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef H_IP_MATCHER_H
|
||||
#define H_IP_MATCHER_H
|
||||
#include <stddef.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
enum IP_TYPE
|
||||
{
|
||||
IPv4,
|
||||
IPv6
|
||||
};
|
||||
|
||||
/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv4<76><34><EFBFBD><EFBFBD> */
|
||||
struct ipv4_range
|
||||
{
|
||||
unsigned int start_ip; /* IP<49><50>Χ<EFBFBD>½<EFBFBD> */
|
||||
unsigned int end_ip; /* IP<49><50>Χ<EFBFBD>Ͻ<EFBFBD> */
|
||||
};
|
||||
|
||||
/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv6<76><36><EFBFBD><EFBFBD> */
|
||||
struct ipv6_range
|
||||
{
|
||||
unsigned int start_ip[4]; /* IP<49><50>Χ<EFBFBD>½<EFBFBD> */
|
||||
unsigned int end_ip[4]; /* IP<49><50>Χ<EFBFBD>Ͻ<EFBFBD> */
|
||||
};
|
||||
|
||||
/* ͨ<>õ<EFBFBD>ip<69><70><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
|
||||
struct ip_rule
|
||||
{
|
||||
enum IP_TYPE type; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͣ<EFBFBD>ipv4<76><34>ipv6 */
|
||||
unsigned int rule_id; /* <20><><EFBFBD><EFBFBD>ID */
|
||||
void* user_tag; /* <20>û<EFBFBD><C3BB>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD><DDA3><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1>ƥ<EFBFBD><C6A5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
|
||||
union
|
||||
{
|
||||
struct ipv4_range ipv4_rule; /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv4<76><34><EFBFBD><EFBFBD>*/
|
||||
struct ipv6_range ipv6_rule; /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv6<76><36><EFBFBD><EFBFBD>*/
|
||||
};
|
||||
};
|
||||
|
||||
/* ͨ<>õĴ<C3B5>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
|
||||
struct ip_data
|
||||
{
|
||||
enum IP_TYPE type; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͣ<EFBFBD>ipv4<76><34>ipv6 */
|
||||
union /* <20><><EFBFBD><EFBFBD>rule_type<70><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݸ<EFBFBD><DDB8><EFBFBD><EFBFBD><EFBFBD>ipv4<76><34><EFBFBD><EFBFBD>ipv6 */
|
||||
{
|
||||
unsigned int ipv4; /* ipv4<76><34><EFBFBD><EFBFBD>*/
|
||||
unsigned int ipv6[4]; /* ipv6<76><36><EFBFBD><EFBFBD>*/
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
|
||||
struct scan_result
|
||||
{
|
||||
unsigned int rule_id; /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ID */
|
||||
void * tag; /* <20>û<EFBFBD><C3BB>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD><DDA3><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1>ƥ<EFBFBD><C6A5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
|
||||
};
|
||||
|
||||
|
||||
struct ip_matcher;
|
||||
|
||||
/*
|
||||
<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD>
|
||||
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
rules[in]<5D><>һ<EFBFBD><D2BB>ip<69><70><EFBFBD><EFBFBD>
|
||||
rule_num[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
mem_use[out]<5D><><EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
|
||||
ipɨ<70><C9A8><EFBFBD><EFBFBD>,<2C><><EFBFBD>ؿ<EFBFBD>ָ<EFBFBD><D6B8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD>ʧ<EFBFBD><CAA7>
|
||||
*/
|
||||
struct ip_matcher* ip_matcher_new(struct ip_rule * rules, size_t rule_num, size_t * mem_use);
|
||||
|
||||
/*
|
||||
<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>ipɨ<70><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ip<69><70><EFBFBD>ݽ<EFBFBD><DDBD><EFBFBD>ɨ<EFBFBD><C9A8>
|
||||
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
matcher[in]<5D><>ipɨ<70><C9A8><EFBFBD><EFBFBD>
|
||||
data[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĵ<EFBFBD>ɨ<EFBFBD><C9A8>ip<69><70><EFBFBD><EFBFBD>
|
||||
result[in]<5D><><EFBFBD><EFBFBD><EFBFBD>ؽ<EFBFBD><D8BD><EFBFBD><EFBFBD>洢<EFBFBD><E6B4A2><EFBFBD><EFBFBD>
|
||||
size[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĵ<EFBFBD>С
|
||||
<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
|
||||
<09><><EFBFBD>н<EFBFBD><D0BD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><=size<7A><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵΪ-1<><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
|
||||
*/
|
||||
int ip_matcher_match(struct ip_matcher* matcher, struct ip_data * data, struct scan_result* result, size_t size);
|
||||
|
||||
/*
|
||||
<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>ipɨ<70><C9A8><EFBFBD><EFBFBD>
|
||||
<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
matcher[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٵ<EFBFBD>ipɨ<70><C9A8><EFBFBD><EFBFBD>ָ<EFBFBD><D6B8>
|
||||
*/
|
||||
void ip_matcher_free(struct ip_matcher* matcher);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* !defined(H_IP_MATCHER_H) */
|
||||
@@ -13,11 +13,10 @@
|
||||
#include <stddef.h>
|
||||
#include <hs/hs.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "uthash/utarray.h"
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
#include "bool_matcher.h"
|
||||
|
||||
struct adpt_hs_compile_data {
|
||||
@@ -69,7 +68,7 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
|
||||
}
|
||||
|
||||
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
|
||||
// log_error("ERROR: Unable to allocate scratch space. Exiting.\n");
|
||||
fprintf(stderr, "ERROR: Unable to allocate scratch space. Exiting.\n");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
@@ -77,14 +76,14 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr
|
||||
for (size_t i = 1; i < nr_worker_threads; i++) {
|
||||
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error("Unable to clone scratch prototype");
|
||||
fprintf(stderr, "Unable to clone scratch prototype");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
|
||||
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error("Unable to query scratch size");
|
||||
fprintf(stderr, "Unable to query scratch size");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
@@ -115,9 +114,8 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
|
||||
literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns,
|
||||
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
if (compile_err) {
|
||||
printf("compile error: %s", compile_err->message);
|
||||
fprintf(stderr, "%s compile error: %s", __func__, compile_err->message);
|
||||
}
|
||||
|
||||
hs_free_compile_error(compile_err);
|
||||
@@ -130,9 +128,8 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
|
||||
regex_cd->ids, NULL, regex_cd->n_patterns,
|
||||
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
if (compile_err) {
|
||||
printf("compile error: %s", compile_err->message);
|
||||
fprintf(stderr, "%s compile error: %s", __func__, compile_err->message);
|
||||
}
|
||||
hs_free_compile_error(compile_err);
|
||||
goto error;
|
||||
@@ -189,7 +186,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
{
|
||||
if ((scan_mode != SCAN_MODE_BLOCK && scan_mode != SCAN_MODE_STREAM) ||
|
||||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
|
||||
/* log_error("input parameters illegal!"); */
|
||||
fprintf(stderr, "%s input parameters illegal!", __func__);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -199,7 +196,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
|
||||
for (size_t i = 0; i < n_expr_array; i++) {
|
||||
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
|
||||
//log_error("the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM);
|
||||
fprintf(stderr, "the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -209,7 +206,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
|
||||
regex_pattern_num++;
|
||||
} else {
|
||||
/* log_error("unknown pattern type: %d\n", expr_array[i].patterns[j].type); */
|
||||
fprintf(stderr, "unknown pattern type: %d\n", expr_array[i].patterns[j].type);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
@@ -288,10 +285,10 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
/* create bool matcher */
|
||||
hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size);
|
||||
if (hs_instance->hs_rt->bm != NULL) {
|
||||
/* log_info("Adapter_hs module: build bool matcher of %u expressions with %u bytes memory.",
|
||||
n_expr_array, mem_size); */
|
||||
fprintf(stdout, "Adapter_hs module: build bool matcher of %u expressions with %u bytes memory.",
|
||||
n_expr_array, mem_size);
|
||||
} else {
|
||||
/* log_error("Adapter_hs module: build bool matcher failed."); */
|
||||
fprintf(stderr, "Adapter_hs module: build bool matcher failed.");
|
||||
goto error;
|
||||
}
|
||||
free(exprs);
|
||||
@@ -299,7 +296,6 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
/* build hs database */
|
||||
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode);
|
||||
if (ret < 0) {
|
||||
//log_error()
|
||||
goto error;
|
||||
}
|
||||
|
||||
@@ -320,7 +316,6 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads
|
||||
|
||||
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type);
|
||||
if (ret < 0) {
|
||||
// log_error
|
||||
goto error;
|
||||
}
|
||||
|
||||
@@ -24,13 +24,14 @@ extern "C"
|
||||
struct adapter_hs;
|
||||
|
||||
/* scan mode */
|
||||
enum {
|
||||
enum scan_mode {
|
||||
SCAN_MODE_BLOCK = 1,
|
||||
SCAN_MODE_STREAM,
|
||||
SCAN_MODE_MAX
|
||||
};
|
||||
|
||||
/* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */
|
||||
enum {
|
||||
enum pattern_type {
|
||||
PATTERN_TYPE_STR = 1,
|
||||
PATTERN_TYPE_REG,
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "maat_table_schema.h"
|
||||
|
||||
@@ -8,24 +8,33 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
|
||||
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
|
||||
|
||||
add_definitions(-fPIC)
|
||||
set(MAAT_SRC maat_api.cpp bool_matcher.cpp adapter_hs.cpp rcu_hash.cpp maat_garbage_collection.cpp)
|
||||
set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp
|
||||
maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
|
||||
|
||||
set(MAAT_SRC_M main.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp
|
||||
maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
|
||||
|
||||
set(LIB_SOURCE_FILES
|
||||
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c)
|
||||
|
||||
include_directories(${PROJECT_SOURCE_DIR}/include/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/deps/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/scanner)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
|
||||
|
||||
# Static Library Output
|
||||
add_library(maat_frame_static STATIC ${MAAT_SRC})
|
||||
#add_dependencies(maat_frame_static hyperscan_static hyperscan_runtime_static)
|
||||
add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES})
|
||||
set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
|
||||
set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
|
||||
set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
|
||||
|
||||
target_link_libraries(maat_frame_static hyperscan_static)
|
||||
target_link_libraries(maat_frame_static hyperscan_runtime_static)
|
||||
target_link_libraries(maat_frame_static adapter-static pthread)
|
||||
|
||||
add_executable(main ${MAAT_SRC_M})
|
||||
target_link_libraries(main maat_frame_static)
|
||||
|
||||
# Shared Library Output
|
||||
#add_library(maat_frame_shared SHARED ${MAAT_SRC})
|
||||
#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
|
||||
#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
|
||||
#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
|
||||
#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
|
||||
@@ -33,7 +42,6 @@ target_link_libraries(maat_frame_static hyperscan_runtime_static)
|
||||
#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
|
||||
|
||||
#target_link_libraries(maat_frame_shared hyperscan_static)
|
||||
#target_link_libraries(maat_frame_shared hyperscan_runtime_static)
|
||||
|
||||
# install
|
||||
set(CMAKE_INSTALL_PREFIX /opt/MESA/)
|
||||
|
||||
@@ -20,6 +20,13 @@ extern "C"
|
||||
|
||||
struct maat_options {
|
||||
size_t nr_worker_threads;
|
||||
int rule_effect_interval_ms;
|
||||
int rule_update_checking_interval_ms;
|
||||
int gc_timeout_ms;
|
||||
int deferred_load_on;
|
||||
enum data_source input_mode;
|
||||
char iris_full_dir[NAME_MAX];
|
||||
char iris_inc_dir[NAME_MAX];
|
||||
};
|
||||
|
||||
#ifdef __cpluscplus
|
||||
|
||||
@@ -18,15 +18,14 @@ extern "C"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define CONFIG_UPDATE_TYPE_NONE 0
|
||||
#define CONFIG_UPDATE_TYPE_FULL 1
|
||||
#define CONFIG_UPDATE_TYPE_INC 2
|
||||
#define CM_UPDATE_TYPE_NONE 0
|
||||
#define CM_UPDATE_TYPE_FULL 1
|
||||
#define CM_UPDATE_TYPE_INC 2
|
||||
|
||||
|
||||
void config_monitor_traverse(uint64_t version, const char *idx_dir,
|
||||
void (*pre_fn)(uint64_t, int, void *),
|
||||
void config_monitor_traverse(long long version, const char *idx_dir,
|
||||
void (*start_fn)(long long, int, void *),
|
||||
int (*update_fn)(const char *, const char *, void *),
|
||||
void (*post_fn)(void *),
|
||||
void (*finish_fn)(void *),
|
||||
void *u_param);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
|
||||
@@ -16,12 +16,47 @@ extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include "rcu_hash.h"
|
||||
|
||||
struct ex_data_runtime;
|
||||
|
||||
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data));
|
||||
/* ex_data_runtime API */
|
||||
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn);
|
||||
|
||||
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
/* ex_data_runtime cache row API */
|
||||
void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row);
|
||||
|
||||
const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index);
|
||||
|
||||
size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
/* set schema API */
|
||||
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
|
||||
|
||||
/* set user_ctx API */
|
||||
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx);
|
||||
|
||||
/* ex_data_runtime ex data API */
|
||||
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len);
|
||||
|
||||
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data);
|
||||
|
||||
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
|
||||
|
||||
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
|
||||
|
||||
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array);
|
||||
|
||||
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_KV_MAP_H_
|
||||
#define _MAAT_KV_MAP_H_
|
||||
#ifndef _MAAT_KV_H_
|
||||
#define _MAAT_KV_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
|
||||
@@ -24,49 +24,67 @@ extern "C"
|
||||
|
||||
struct maat_runtime {
|
||||
/* maat_runtime can be created and destroy dynamic, so need version info */
|
||||
uint64_t version;
|
||||
long long version;
|
||||
|
||||
time_t last_update_time;
|
||||
|
||||
struct maat_table_runtime_manager *table_rt_mgr;
|
||||
struct table_runtime_manager *table_rt_mgr;
|
||||
size_t max_table_num;
|
||||
|
||||
int max_thread_num;
|
||||
size_t max_thread_num;
|
||||
uint32_t rule_num;
|
||||
};
|
||||
|
||||
enum rule_import_type {
|
||||
RULE_IMPORT_TYPE_IRIS = 1,
|
||||
RULE_IMPORT_TYPE_MAX
|
||||
enum data_source {
|
||||
DATA_SOURCE_NONE = 0,
|
||||
DATA_SOURCE_IRIS_FILE
|
||||
};
|
||||
|
||||
struct rule_import_iris_ctx {
|
||||
struct source_iris_ctx {
|
||||
char inc_dir[NAME_MAX];
|
||||
char full_dir[NAME_MAX];
|
||||
};
|
||||
|
||||
struct maat {
|
||||
struct maat_runtime *maat_rt;
|
||||
struct maat_runtime *rebuilding_maat_rt; //TODO: creating
|
||||
//struct maat_garbage_collector *gc;
|
||||
struct maat_table_manager *table_mgr;
|
||||
char instance_name[NAME_MAX];
|
||||
|
||||
enum rule_import_type rule_import_type;
|
||||
struct maat_runtime *maat_rt;
|
||||
struct maat_runtime *creating_maat_rt;
|
||||
|
||||
struct table_schema_manager *table_schema_mgr;
|
||||
|
||||
enum data_source input_mode;
|
||||
union {
|
||||
struct rule_import_iris_ctx iris_ctx;
|
||||
struct source_iris_ctx iris_ctx;
|
||||
};
|
||||
|
||||
int deferred_load;
|
||||
|
||||
int is_running;
|
||||
pthread_mutex_t background_update_mutex;
|
||||
int nr_worker_thread;
|
||||
|
||||
uint64_t maat_version;
|
||||
uint64_t last_full_version;
|
||||
long long maat_version;
|
||||
long long last_full_version;
|
||||
pthread_t cfg_mon_thread;
|
||||
|
||||
int rule_effect_interval_ms;
|
||||
int rule_update_checking_interval_ms;
|
||||
int gc_timeout_ms; //garbage collection timeout_ms;
|
||||
|
||||
struct maat_garbage_bin *garbage_bin;
|
||||
};
|
||||
|
||||
void maat_start_cb(long long new_version, int update_type, void *u_para);
|
||||
|
||||
int maat_update_cb(const char *table_name, const char *line, void *u_para);
|
||||
|
||||
void maat_finish_cb(void *u_para);
|
||||
|
||||
void *rule_monitor_loop(void *arg);
|
||||
|
||||
void maat_read_full_config(struct maat *maat_instance);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -19,19 +19,51 @@ extern "C"
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_garbage_collection.h"
|
||||
|
||||
struct maat_table_item;
|
||||
struct maat_table_runtime;
|
||||
struct maat_table_runtime_manager;
|
||||
struct table_item;
|
||||
struct table_runtime;
|
||||
struct table_runtime_manager;
|
||||
|
||||
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* bin);
|
||||
/* table runtime manager API */
|
||||
struct table_runtime_manager *table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
|
||||
struct maat_garbage_bin *bin);
|
||||
|
||||
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr);
|
||||
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
|
||||
|
||||
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id);
|
||||
/* table runtime API */
|
||||
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id);
|
||||
|
||||
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt);
|
||||
size_t table_runtime_rule_count(struct table_runtime *table_rt);
|
||||
|
||||
void maat_table_runtime_item_add(struct maat_table_runtime *table_rt, struct maat_table_item *table_item);
|
||||
enum table_type table_runtime_get_type(struct table_runtime* table_rt);
|
||||
|
||||
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *line, struct table_item *table_item);
|
||||
|
||||
/**
|
||||
* @brief if table_runtime is updating
|
||||
*
|
||||
* @retval 1(yes) 0(no)
|
||||
*/
|
||||
int table_runtime_updating_flag(struct table_runtime *table_rt);
|
||||
|
||||
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread);
|
||||
|
||||
/* table runtime scan API */
|
||||
int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len,
|
||||
int results[], size_t *n_result);
|
||||
|
||||
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
|
||||
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len,
|
||||
int result[], size_t n_result);
|
||||
void table_runtime_stream_close(struct table_runtime *table_rt);
|
||||
|
||||
/* plugin table runtime API */
|
||||
size_t plugin_table_runtime_cached_row_count(struct table_runtime *table_rt);
|
||||
|
||||
const char* plugin_table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq);
|
||||
|
||||
void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len);
|
||||
|
||||
void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
|
||||
@@ -18,27 +18,152 @@ extern "C"
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
#include "maat/maat.h"
|
||||
#include "adapter_hs.h"
|
||||
|
||||
enum maat_table_type {
|
||||
#define MAX_DISTRICT_STR 128
|
||||
#define MAX_IP_STR 128
|
||||
#define MAX_KEYWORDS_STR 1024
|
||||
|
||||
enum table_type {
|
||||
TABLE_TYPE_EXPR = 0,
|
||||
TABLE_TYPE_EXPR_PLUS,
|
||||
TABLE_TYPE_IP,
|
||||
TABLE_TYPE_PLUGIN,
|
||||
TABLE_TYPE_IP_PLUGIN,
|
||||
TABLE_TYPE_MAX
|
||||
};
|
||||
|
||||
struct maat_table_schema;
|
||||
struct maat_table_manager;
|
||||
struct maat_table_item;
|
||||
enum expr_type {
|
||||
EXPR_TYPE_STRING = 1,
|
||||
EXPR_TYPE_REGEX,
|
||||
EXPR_TYPE_AND,
|
||||
EXPR_TYPE_OFFSET,
|
||||
EXPR_TYPE_MAX
|
||||
};
|
||||
|
||||
struct maat_table_manager *maat_table_manager_create(sds table_info_path);
|
||||
void maat_table_manager_destroy(struct maat_table_manager *table_mgr);
|
||||
enum match_method {
|
||||
MATCH_METHOD_SUB=0,
|
||||
MATCH_METHOD_RIGHT,
|
||||
MATCH_METHOD_LEFT,
|
||||
MATCH_METHOD_COMPLETE,
|
||||
MATCH_METHOD_MAX
|
||||
};
|
||||
|
||||
int maat_table_manager_get_table_id(struct maat_table_manager* table_mgr, sds table_name);
|
||||
enum maat_table_type maat_table_manager_get_table_type(struct maat_table_manager *table_mgr, int id);
|
||||
struct expr_item {
|
||||
int item_id;
|
||||
int group_id;
|
||||
char district[MAX_DISTRICT_STR];
|
||||
char keywords[MAX_KEYWORDS_STR];
|
||||
enum expr_type expr_type;
|
||||
enum match_method match_method;
|
||||
int is_hexbin;
|
||||
int is_case_sensitive;
|
||||
int is_valid;
|
||||
|
||||
size_t maat_table_manager_get_size(struct maat_table_manager* table_mgr);
|
||||
//rule_tag; 只存在schema里
|
||||
//int have_exdata;
|
||||
//struct ex_data *ex_data; //hash表
|
||||
};
|
||||
|
||||
struct maat_table_item *maat_table_line_to_item(sds line, struct maat_table_schema *table_schema);
|
||||
struct plugin_item {
|
||||
char key[MAX_KEYWORDS_STR];
|
||||
size_t key_len;
|
||||
int is_valid;
|
||||
};
|
||||
|
||||
struct ip_plugin_item {
|
||||
int item_id;
|
||||
int ip_type;
|
||||
char start_ip[MAX_IP_STR];
|
||||
char end_ip[MAX_IP_STR];
|
||||
int is_valid;
|
||||
int rule_tag;
|
||||
int have_exdata;
|
||||
void *ex_data;
|
||||
};
|
||||
|
||||
struct table_item {
|
||||
enum table_type table_type;
|
||||
union {
|
||||
struct expr_item expr_item;
|
||||
struct plugin_item plugin_item;
|
||||
struct ip_plugin_item ip_plugin_item;
|
||||
};
|
||||
};
|
||||
|
||||
struct plugin_table_callback_schema
|
||||
{
|
||||
maat_start_callback_t *start;
|
||||
maat_update_callback_t *update;
|
||||
maat_finish_callback_t *finish;
|
||||
void* u_para;
|
||||
};
|
||||
|
||||
struct ex_data_schema
|
||||
{
|
||||
maat_plugin_ex_new_func_t *new_func;
|
||||
maat_plugin_ex_free_func_t *free_func;
|
||||
maat_plugin_ex_dup_func_t *dup_func;
|
||||
//Maat_plugin_EX_key2index_func_t* key2index_func;
|
||||
long argl;
|
||||
void *argp;
|
||||
int set_flag;
|
||||
};
|
||||
|
||||
struct table_schema;
|
||||
struct table_schema_manager;
|
||||
|
||||
/* table schema manager API */
|
||||
struct table_schema_manager *table_schema_manager_create(const char *table_info_path);
|
||||
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr);
|
||||
|
||||
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name);
|
||||
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
|
||||
|
||||
size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr);
|
||||
|
||||
void table_schema_manager_all_plugin_cb_start(struct table_schema_manager* table_schema_mgr, int update_type);
|
||||
void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr);
|
||||
|
||||
/* table schema generic API */
|
||||
struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);
|
||||
|
||||
enum table_type table_schema_get_table_type(struct table_schema *table_schema);
|
||||
|
||||
int table_schema_get_table_id(struct table_schema *table_schema);
|
||||
|
||||
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema);
|
||||
|
||||
/* expr table schema API */
|
||||
enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
|
||||
|
||||
/* plugin table schema API */
|
||||
int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
|
||||
maat_plugin_ex_new_func_t *new_func,
|
||||
maat_plugin_ex_free_func_t *free_func,
|
||||
maat_plugin_ex_dup_func_t *dup_func,
|
||||
long argl, void *argp);
|
||||
|
||||
struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema);
|
||||
/**
|
||||
* @brief if plugin table schema's ex data schema set
|
||||
*
|
||||
* @retval 1(already Set) 0(Not set yet)
|
||||
*/
|
||||
int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema);
|
||||
|
||||
int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id,
|
||||
maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
|
||||
maat_update_callback_t *update,//table line ,u_para
|
||||
maat_finish_callback_t *finish,//u_para
|
||||
void *u_para);
|
||||
/**
|
||||
* @brief the number of callback function stored in plugin table schema
|
||||
*/
|
||||
size_t plugin_table_schema_callback_count(struct table_schema *table_schema);
|
||||
|
||||
void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
|
||||
@@ -19,11 +19,28 @@ extern "C"
|
||||
#include <stdlib.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
#define TRUE 1
|
||||
#define FALSE 0
|
||||
|
||||
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
|
||||
#ifndef MAX
|
||||
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
|
||||
#endif
|
||||
|
||||
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len);
|
||||
#ifndef MIN
|
||||
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
|
||||
#endif
|
||||
|
||||
char *maat_strdup(const char *s);
|
||||
|
||||
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
|
||||
|
||||
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz);
|
||||
|
||||
char *strtok_r_esc(char *s, const char delim, char **save_ptr);
|
||||
|
||||
char *str_unescape_and(char *s);
|
||||
|
||||
char *str_unescape(char *s);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
|
||||
@@ -18,16 +18,20 @@ extern "C"
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
|
||||
typedef void rcu_hash_data_free_fn(void *user_ctx, void *data);
|
||||
|
||||
/* rcu hash table */
|
||||
struct rcu_hash_table;
|
||||
|
||||
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data));
|
||||
struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn);
|
||||
|
||||
void rcu_hash_free(struct rcu_hash_table *htable);
|
||||
|
||||
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx);
|
||||
|
||||
/**
|
||||
* @brief the data added just in updating stage
|
||||
* after call rcu_hash_commit, it in effective stage
|
||||
* @brief just means add to the updating nodes
|
||||
* after call rcu_hash_commit, they become effective nodes
|
||||
*/
|
||||
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data);
|
||||
|
||||
@@ -44,14 +48,24 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
|
||||
*/
|
||||
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
|
||||
|
||||
size_t rcu_hash_counts(struct rcu_hash_table *htable);
|
||||
|
||||
size_t rcu_hash_count(struct rcu_hash_table *htable);
|
||||
|
||||
/**
|
||||
* @brief make add/del effective
|
||||
*/
|
||||
void rcu_hash_commit(struct rcu_hash_table *htable);
|
||||
|
||||
size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable);
|
||||
|
||||
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array);
|
||||
|
||||
/**
|
||||
* @brief check if rcu hash table is updating
|
||||
*
|
||||
* @retval 1 means htable is updating, otherwise 0
|
||||
*/
|
||||
int rcu_hash_updating_flag(struct rcu_hash_table *htable);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
229
src/maat_api.cpp
229
src/maat_api.cpp
@@ -9,19 +9,27 @@
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
#include "maat/maat.h"
|
||||
#include "maat_rule.h"
|
||||
#include "maat_common.h"
|
||||
#include "maat_kv.h"
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "sds/sds.h"
|
||||
#include "maat_config_monitor.h"
|
||||
|
||||
struct maat_options* maat_options_new(void)
|
||||
{
|
||||
struct maat_options *options = ALLOC(struct maat_options, 1);
|
||||
|
||||
options->nr_worker_threads = 1;
|
||||
options->deferred_load_on = 0;
|
||||
options->rule_effect_interval_ms = 60 * 1000;
|
||||
options->rule_update_checking_interval_ms = 1 * 1000;
|
||||
options->gc_timeout_ms = 10 * 1000;
|
||||
options->input_mode = DATA_SOURCE_NONE;
|
||||
|
||||
return options;
|
||||
}
|
||||
@@ -33,25 +41,129 @@ int maat_options_set_worker_thread_number(struct maat_options *opts, size_t n_wo
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct maat *maat_new(struct maat_options opts, const char *table_info_path)
|
||||
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms)
|
||||
{
|
||||
opts->rule_effect_interval_ms = interval_ms;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms)
|
||||
{
|
||||
opts->rule_update_checking_interval_ms = interval_ms;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
|
||||
{
|
||||
opts->gc_timeout_ms = interval_ms;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_options_set_deferred_load_on(struct maat_options *opts)
|
||||
{
|
||||
opts->deferred_load_on = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir)
|
||||
{
|
||||
if (strlen(full_dir) >= NAME_MAX) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(opts->iris_full_dir, full_dir, strlen(full_dir));
|
||||
opts->input_mode = DATA_SOURCE_IRIS_FILE;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir)
|
||||
{
|
||||
if (strlen(inc_dir) >= NAME_MAX) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
memcpy(opts->iris_inc_dir, inc_dir, strlen(inc_dir));
|
||||
opts->input_mode = DATA_SOURCE_IRIS_FILE;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void maat_read_full_config(struct maat *maat_instance)
|
||||
{
|
||||
switch (maat_instance->input_mode) {
|
||||
case DATA_SOURCE_IRIS_FILE:
|
||||
config_monitor_traverse(maat_instance->maat_version,
|
||||
maat_instance->iris_ctx.full_dir,
|
||||
maat_start_cb,
|
||||
maat_update_cb,
|
||||
maat_finish_cb,
|
||||
maat_instance);
|
||||
if (NULL == maat_instance->creating_maat_rt) {
|
||||
fprintf(stderr, "At initiation: NO effective rule in %s.",
|
||||
maat_instance->iris_ctx.full_dir);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
maat_instance->maat_rt = maat_instance->creating_maat_rt;
|
||||
maat_instance->creating_maat_rt = NULL;
|
||||
maat_instance->is_running = 1;
|
||||
if (maat_instance->maat_rt != NULL) {
|
||||
maat_instance->maat_version = maat_instance->maat_rt->version;
|
||||
maat_instance->last_full_version = maat_instance->maat_rt->version;
|
||||
}
|
||||
}
|
||||
|
||||
struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
|
||||
{
|
||||
#if 0
|
||||
if (NULL == table_info_path) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int garbage_gc_timeout_s = 0;
|
||||
struct maat *maat_instance = ALLOC(struct maat, 1);
|
||||
|
||||
maat_instance->table_mgr = maat_table_manager_create(table_info_path);
|
||||
maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path);
|
||||
if (NULL == maat_instance->table_schema_mgr) {
|
||||
goto failed;
|
||||
}
|
||||
|
||||
maat_instance->input_mode = opts->input_mode;
|
||||
switch (maat_instance->input_mode) {
|
||||
case DATA_SOURCE_IRIS_FILE:
|
||||
memcpy(maat_instance->iris_ctx.full_dir, opts->iris_full_dir, strlen(opts->iris_full_dir));
|
||||
memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_inc_dir, strlen(opts->iris_inc_dir));
|
||||
break;
|
||||
default:
|
||||
fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode);
|
||||
goto failed;
|
||||
}
|
||||
|
||||
maat_instance->is_running = 0;
|
||||
maat_instance->maat_version = 0;
|
||||
maat_instance->last_full_version = 0;
|
||||
maat_instance->nr_worker_thread = opts.nr_worker_threads;
|
||||
maat_instance->nr_worker_thread = opts->nr_worker_threads;
|
||||
maat_instance->rule_effect_interval_ms = opts->rule_effect_interval_ms;
|
||||
maat_instance->gc_timeout_ms = opts->gc_timeout_ms;
|
||||
maat_instance->deferred_load = opts->deferred_load_on;
|
||||
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
|
||||
(maat_instance->gc_timeout_ms / 1000);
|
||||
maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
|
||||
pthread_mutex_init(&(maat_instance->background_update_mutex), NULL);
|
||||
|
||||
if (0 == maat_instance->deferred_load) {
|
||||
maat_read_full_config(maat_instance);
|
||||
}
|
||||
|
||||
pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void*)maat_instance);
|
||||
|
||||
return maat_instance;
|
||||
#endif
|
||||
failed:
|
||||
free(maat_instance);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -64,9 +176,98 @@ void maat_free(struct maat *maat_instance)
|
||||
return;
|
||||
}
|
||||
|
||||
int maat_table_get_id(struct maat *instance, const char *table_name)
|
||||
int maat_table_get_id(struct maat *maat_instance, const char *table_name)
|
||||
{
|
||||
int table_id = -1;
|
||||
|
||||
struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr;
|
||||
table_id = table_schema_manager_get_table_id(table_schema_mgr, table_name);
|
||||
|
||||
return table_id;
|
||||
}
|
||||
|
||||
int maat_table_callback_register(struct maat *maat_instance, int table_id,
|
||||
maat_start_callback_t *start,
|
||||
maat_update_callback_t *update,
|
||||
maat_finish_callback_t *finish,
|
||||
void *u_para)
|
||||
{
|
||||
int ret = -1;
|
||||
|
||||
pthread_mutex_lock(&(maat_instance->background_update_mutex));
|
||||
ret = plugin_table_schema_add_callback(maat_instance->table_schema_mgr, table_id, start, update, finish, u_para);
|
||||
if (ret < 0) {
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!maat_instance->maat_rt) {
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct table_runtime *table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
|
||||
size_t row_count = plugin_table_runtime_cached_row_count(table_rt);
|
||||
if (row_count > 0) {
|
||||
if (start != NULL) {
|
||||
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < row_count; i++) {
|
||||
const char *line = plugin_table_runtime_get_cached_row(table_rt, i);
|
||||
if (NULL == line) {
|
||||
break;
|
||||
}
|
||||
|
||||
update(table_id, line, u_para);
|
||||
}
|
||||
|
||||
if (finish != NULL) {
|
||||
finish(u_para);
|
||||
}
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id,
|
||||
maat_plugin_ex_new_func_t *new_func,
|
||||
maat_plugin_ex_free_func_t *free_func,
|
||||
maat_plugin_ex_dup_func_t *dup_func,
|
||||
long argl, void *argp)
|
||||
{
|
||||
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
|
||||
pthread_mutex_lock(&(maat_instance->background_update_mutex));
|
||||
int ret = plugin_table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, argl, argp);
|
||||
if (ret < 0) {
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct table_runtime *table_rt = NULL;
|
||||
if (maat_instance->maat_rt != NULL) {
|
||||
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
|
||||
plugin_table_runtime_commit_ex_data_schema(table_rt, table_schema);
|
||||
}
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
|
||||
const char *key, size_t key_len)
|
||||
{
|
||||
struct maat_runtime *maat_rt = maat_instance->maat_rt;
|
||||
if (NULL == maat_rt) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
|
||||
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
|
||||
|
||||
return plugin_table_runtime_get_ex_data(table_rt, table_schema, key, key_len);
|
||||
}
|
||||
|
||||
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
|
||||
@@ -87,16 +288,10 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
|
||||
const char *data, size_t data_len, int results[], size_t *n_results,
|
||||
struct maat_state *state)
|
||||
{
|
||||
#if 0
|
||||
if (NULL == maat_instance || NULL == data ) {
|
||||
return -1;
|
||||
}
|
||||
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr;
|
||||
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
|
||||
|
||||
struct maat_runtime *maat_rt = maat_instance->maat_rt;
|
||||
|
||||
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, table_id);
|
||||
#endif
|
||||
return 0;
|
||||
return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_results);
|
||||
}
|
||||
|
||||
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
|
||||
|
||||
@@ -13,9 +13,11 @@
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
|
||||
#include "maat_config_monitor.h"
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
|
||||
#define CM_MAX_TABLE_NUM 256
|
||||
#define MAX_CONFIG_LINE (1024 * 16)
|
||||
@@ -28,17 +30,6 @@ struct cm_table_info_t
|
||||
char encryp_algorithm[NAME_MAX];
|
||||
};
|
||||
|
||||
/*
|
||||
@brief check if rule file updated
|
||||
@retval 0 -> CONFIG_UPDATE_TYPE_NONE
|
||||
1 -> CONFIG_UPDATE_TYPE_FULL
|
||||
2 -> CONFIG_UPDATE_TYPE_INC
|
||||
*/
|
||||
int validate_update_happened(uint64_t current_version, const char *idx_dir, char **idx_paths[], size_t *n_idx_paths)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int size)
|
||||
{
|
||||
int ret = 0;
|
||||
@@ -55,7 +46,7 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
|
||||
,idx[i].cfg_path
|
||||
,idx[i].encryp_algorithm);
|
||||
//jump over empty line
|
||||
if (!(ret==3||ret==4)||idx[i].cfg_num==0){
|
||||
if (!(ret == 3 || ret == 4) || idx[i].cfg_num == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -77,17 +68,15 @@ int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int si
|
||||
return i;
|
||||
}
|
||||
|
||||
const char* path2filename(const char*path)
|
||||
const char *path2filename(const char *path)
|
||||
{
|
||||
int i=0;
|
||||
for(i=strlen(path);i>0;i--)
|
||||
{
|
||||
if(path[i]=='/')
|
||||
{
|
||||
int i = 0;
|
||||
for (i = strlen(path);i > 0;i--) {
|
||||
if (path[i]=='/') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return path+i+1;
|
||||
return path + i + 1;
|
||||
}
|
||||
|
||||
char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size)
|
||||
@@ -96,9 +85,9 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset
|
||||
const char* p;
|
||||
|
||||
//search for CRLF, aka CR '\r'(old Mac), LF '\n'(UNIX) or CRLF"\r\n" (Windows)
|
||||
p = (const char*)memchr(buff+*offset, '\n', buff_size-*offset);
|
||||
p = (const char *)memchr(buff + *offset, '\n', buff_size - *offset);
|
||||
if (p == NULL) { // NOT "\n" or "\r\n"
|
||||
p = (const char*)memchr(buff+*offset, '\r', buff_size-*offset);
|
||||
p = (const char *)memchr(buff + *offset, '\r', buff_size - *offset);
|
||||
}
|
||||
|
||||
if (p != NULL) { //point to next character
|
||||
@@ -108,7 +97,7 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset
|
||||
}
|
||||
|
||||
this_offset = p - (buff + *offset);
|
||||
memcpy(line, buff + *offset, MIN(this_offset, line_size-1));
|
||||
memcpy(line, buff + *offset, MIN(this_offset, line_size - 1));
|
||||
|
||||
*offset += this_offset;
|
||||
line[MIN(this_offset, line_size - 1)] = '\0';
|
||||
@@ -122,7 +111,6 @@ int cm_read_table_file(struct cm_table_info_t* index,
|
||||
{
|
||||
int cfg_num = 0,i =0;
|
||||
int ret = 0;
|
||||
char error_string[NAME_MAX];
|
||||
char line[MAX_CONFIG_LINE]={0};
|
||||
char *ret_str=NULL;
|
||||
char *table_file_buff=NULL;
|
||||
@@ -144,16 +132,15 @@ int cm_read_table_file(struct cm_table_info_t* index,
|
||||
}
|
||||
|
||||
for (i = 0; i < cfg_num; i++) {
|
||||
line[sizeof(line)-1]='\0';
|
||||
line[sizeof(line) - 1] = '\0';
|
||||
|
||||
ret_str=read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
|
||||
|
||||
if(ret_str==NULL) {
|
||||
ret_str = read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
|
||||
if (ret_str == NULL) {
|
||||
//log_error
|
||||
break;
|
||||
}
|
||||
|
||||
if(line[sizeof(line)-1]!='\0') {
|
||||
if(line[sizeof(line) - 1] != '\0') {
|
||||
//log_error
|
||||
continue;
|
||||
}
|
||||
@@ -169,32 +156,174 @@ int cm_read_table_file(struct cm_table_info_t* index,
|
||||
return 0;
|
||||
}
|
||||
|
||||
void config_monitor_traverse(uint64_t current_version, const char *idx_dir,
|
||||
void (*start_fn)(uint64_t, int, void *),
|
||||
//replacement of glibc scandir, to adapt dictator malloc wrap
|
||||
#define ENLARGE_STEP 1024
|
||||
int my_scandir(const char *dir, struct dirent ***namelist,
|
||||
int(*filter)(const struct dirent *),
|
||||
int(*compar)(const void *, const void *))
|
||||
{
|
||||
int n = 0;
|
||||
int DIR_ENT_SIZE = ENLARGE_STEP;
|
||||
struct dirent entry, *result;
|
||||
|
||||
if ((NULL == dir) || (NULL == namelist)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
DIR *od = opendir(dir);
|
||||
if (NULL == od) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct dirent **list = ALLOC(struct dirent *, DIR_ENT_SIZE);
|
||||
while (0 == readdir_r(od, &entry, &result)) {
|
||||
if (NULL == result) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (filter && !filter(&entry)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
struct dirent *p = ALLOC(struct dirent, 1);
|
||||
memcpy((void *)p, (void *)(&entry), sizeof(struct dirent));
|
||||
list[n] = p;
|
||||
|
||||
n++;
|
||||
if (n >= DIR_ENT_SIZE) {
|
||||
DIR_ENT_SIZE += ENLARGE_STEP;
|
||||
list = (struct dirent **)realloc((void*)list, DIR_ENT_SIZE * sizeof(struct dirent *));
|
||||
}
|
||||
}
|
||||
|
||||
closedir(od);
|
||||
*namelist = list;
|
||||
|
||||
if (compar) {
|
||||
qsort((void *)*namelist, n, sizeof(struct dirent *), compar);
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
int filter_fn(const struct dirent *ent)
|
||||
{
|
||||
return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0 ||
|
||||
strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
|
||||
}
|
||||
|
||||
int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num)
|
||||
{
|
||||
struct dirent **namelist = NULL;
|
||||
int update_type = CM_UPDATE_TYPE_NONE;
|
||||
|
||||
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
|
||||
if (n < 0) {
|
||||
//log_error("scan dir error");
|
||||
return update_type;
|
||||
}
|
||||
|
||||
long long latest_full_version = 0;
|
||||
long long latest_inc_version = 0;
|
||||
int full_file_idx = 0;
|
||||
int *inc_file_idx = ALLOC(int, n);
|
||||
int inc_idx_num = 0;
|
||||
int i = 0;
|
||||
for (i = 0; i < n; i++) {
|
||||
long long config_seq = 0;
|
||||
char update_str[32] = {0};
|
||||
|
||||
if ((strcmp(namelist[i]->d_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strlen(namelist[i]->d_name) > 42) {
|
||||
//log_error("config file %s filename too long,should like full_config_index.00000000000000000001",namelist[i]->d_name);
|
||||
continue;
|
||||
}
|
||||
|
||||
int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq);
|
||||
if (ret != 2) {
|
||||
//log_error("config file %s filename error,should like full_config_index.00000000000000000001",namelist[i]->d_name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (strncasecmp(update_str, "full", strlen(update_str)) == 0) {
|
||||
if (config_seq > latest_full_version) {
|
||||
latest_full_version = config_seq;
|
||||
full_file_idx = i;
|
||||
}
|
||||
} else if(strncasecmp(update_str,"inc",strlen(update_str))==0) {
|
||||
if (config_seq > current_version) {
|
||||
inc_file_idx[inc_idx_num] = i;
|
||||
inc_idx_num++;
|
||||
if (config_seq > latest_inc_version) {
|
||||
latest_inc_version = config_seq;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//log_error("config file %s,not full or inc config",namelist[i]->d_name);
|
||||
}
|
||||
}
|
||||
|
||||
size_t path_len = 0;
|
||||
//full update
|
||||
if (latest_full_version > current_version) {
|
||||
*idx_path = (char **)malloc(sizeof(char **));
|
||||
path_len = strlen(file_dir) + strlen(namelist[full_file_idx]->d_name) + 1 + 1;
|
||||
(*idx_path)[0] = (char *)malloc(path_len);
|
||||
snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name);
|
||||
*idx_num = 1;
|
||||
update_type = CM_UPDATE_TYPE_FULL;
|
||||
} else if (latest_inc_version > current_version) {
|
||||
//inc update,it's possible that do inc after full update in this function,but we'll process it at next loop.
|
||||
*idx_path = (char **)malloc(sizeof(char **) * inc_idx_num);
|
||||
for (i = 0; i < inc_idx_num; i++) {
|
||||
path_len = strlen(file_dir) + strlen(namelist[inc_file_idx[i]]->d_name) + 1 + 1;
|
||||
(*idx_path)[i] = (char *)malloc(path_len);
|
||||
snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name);
|
||||
}
|
||||
*idx_num = inc_idx_num;
|
||||
update_type = CM_UPDATE_TYPE_INC;
|
||||
} else {
|
||||
update_type = CM_UPDATE_TYPE_NONE;
|
||||
}
|
||||
|
||||
free(inc_file_idx);
|
||||
for (i = 0; i < n; i++) {
|
||||
free(namelist[i]);
|
||||
}
|
||||
free(namelist);
|
||||
|
||||
return update_type;
|
||||
}
|
||||
|
||||
void config_monitor_traverse(long long current_version, const char *idx_dir,
|
||||
void (*start_fn)(long long, int, void *),
|
||||
int (*update_fn)(const char *, const char *, void *),
|
||||
void (*finish_fn)(void *),
|
||||
void *u_param)
|
||||
{
|
||||
size_t i = 0;
|
||||
uint64_t new_version = 0;
|
||||
long long new_version = 0;
|
||||
char **idx_path_array = NULL;
|
||||
size_t idx_path_num = 0;
|
||||
struct cm_table_info_t table_array[CM_MAX_TABLE_NUM];
|
||||
|
||||
memset(table_array, 0, sizeof(table_array));
|
||||
|
||||
int update_type = validate_update_happened(current_version, idx_dir, &idx_path_array, &idx_path_num);
|
||||
if (update_type != CONFIG_UPDATE_TYPE_NONE) {
|
||||
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num);
|
||||
if (update_type != CM_UPDATE_TYPE_NONE) {
|
||||
for (i = 0; i < idx_path_num; i++) {
|
||||
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
|
||||
if (table_num < 0) {
|
||||
//log_error
|
||||
//log_error luis
|
||||
break;
|
||||
}
|
||||
|
||||
char str_not_care[256] = {0};
|
||||
const char* table_filename = path2filename(idx_path_array[i]);
|
||||
sscanf(table_filename,"%[a-zA-Z]_config_index.%lld", str_not_care, &new_version);
|
||||
const char *table_filename = path2filename(idx_path_array[i]);
|
||||
sscanf(table_filename, "%[a-zA-Z]_config_index.%lld", str_not_care, &new_version);
|
||||
if (start_fn != NULL) {
|
||||
start_fn(new_version, update_type, u_param);
|
||||
}
|
||||
|
||||
@@ -9,53 +9,50 @@
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
#include "uthash/utarray.h"
|
||||
#include "maat_rhash.h"
|
||||
#include "rcu_hash.h"
|
||||
#include "utils.h"
|
||||
#include "maat_utils.h"
|
||||
|
||||
typedef void ex_data_schema_new_func_t(int table_id, const char *key, const char *table_line, void* ad, long argl, void *argp);
|
||||
typedef void ex_data_schema_free_func_t(int table_id, void* ad, long argl, void *argp);
|
||||
typedef void ex_data_schema_dup_func_t(int table_id, void *to, void *from, long argl, void *argp);
|
||||
/*
|
||||
struct ex_data_schema
|
||||
{
|
||||
ex_data_schema_new_func_t* new_func;
|
||||
ex_data_schema_free_func_t* free_func;
|
||||
ex_data_schema_dup_func_t* dup_func;
|
||||
//Maat_plugin_EX_key2index_func_t* key2index_func;
|
||||
long argl;
|
||||
void *argp;
|
||||
}; */
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_ex_data.h"
|
||||
|
||||
struct ex_data_runtime {
|
||||
UT_array *cache_rows;
|
||||
size_t cache_row_num;
|
||||
size_t cache_size;
|
||||
|
||||
struct rhash_table *htable;
|
||||
//const struct ex_data_schema* ex_schema;
|
||||
struct rcu_hash_table *htable;
|
||||
struct ex_data_schema *ex_schema;
|
||||
int table_id;
|
||||
};
|
||||
|
||||
void cache_row_free(void*p)
|
||||
void cache_row_free(void *p)
|
||||
{
|
||||
free(*(char**)p);
|
||||
free(*(char **)p);
|
||||
}
|
||||
|
||||
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
|
||||
|
||||
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data))
|
||||
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn)
|
||||
{
|
||||
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
|
||||
|
||||
utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd);
|
||||
ex_data_rt->htable = rhash_table_create(data_free);
|
||||
ex_data_rt->htable = rcu_hash_new(data_free_fn);
|
||||
ex_data_rt->table_id = table_id;
|
||||
|
||||
return ex_data_rt;
|
||||
}
|
||||
|
||||
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
if (NULL == ex_data_rt) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (ex_data_rt->cache_rows != NULL) {
|
||||
utarray_free(ex_data_rt->cache_rows);
|
||||
ex_data_rt->cache_rows=NULL;
|
||||
@@ -63,8 +60,128 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
|
||||
}
|
||||
|
||||
if (ex_data_rt->htable != NULL) {
|
||||
rhash_table_destroy(ex_data_rt->htable);
|
||||
rcu_hash_free(ex_data_rt->htable);
|
||||
}
|
||||
|
||||
free(ex_data_rt);
|
||||
}
|
||||
|
||||
void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
rcu_hash_commit(ex_data_rt->htable);
|
||||
}
|
||||
|
||||
void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row)
|
||||
{
|
||||
size_t len = strlen(row) + 1;
|
||||
char* row_copy = ALLOC(char, len);
|
||||
|
||||
memcpy(row_copy, row, len);
|
||||
ex_data_rt->cache_size += len;
|
||||
utarray_push_back(ex_data_rt->cache_rows, &row_copy);
|
||||
ex_data_rt->cache_row_num++;
|
||||
}
|
||||
|
||||
const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index)
|
||||
{
|
||||
const char** row = NULL;
|
||||
row = (const char **)utarray_eltptr(ex_data_rt->cache_rows, index);
|
||||
return *row;
|
||||
}
|
||||
|
||||
void ex_data_runtime_clear_cached_row(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
utarray_free(ex_data_rt->cache_rows);
|
||||
ex_data_rt->cache_rows = NULL;
|
||||
ex_data_rt->cache_row_num = 0;
|
||||
ex_data_rt->cache_size = 0;
|
||||
}
|
||||
|
||||
size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
if (NULL == ex_data_rt) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return ex_data_rt->cache_row_num;
|
||||
}
|
||||
|
||||
void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
utarray_free(ex_data_rt->cache_rows);
|
||||
ex_data_rt->cache_rows = NULL;
|
||||
ex_data_rt->cache_row_num = 0;
|
||||
ex_data_rt->cache_size = 0;
|
||||
}
|
||||
|
||||
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema)
|
||||
{
|
||||
ex_data_rt->ex_schema = schema;
|
||||
}
|
||||
|
||||
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx)
|
||||
{
|
||||
rcu_hash_set_user_ctx(ex_data_rt->htable, user_ctx);
|
||||
}
|
||||
|
||||
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len)
|
||||
{
|
||||
void *ex_data = NULL;
|
||||
struct ex_data_schema *ex_schema = ex_data_rt->ex_schema;
|
||||
ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, ex_schema->argl, ex_schema->argp);
|
||||
|
||||
return ex_data;
|
||||
}
|
||||
|
||||
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data)
|
||||
{
|
||||
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
|
||||
if (tmp_data != NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
rcu_hash_add(ex_data_rt->htable, key, key_len, data);
|
||||
}
|
||||
|
||||
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
|
||||
{
|
||||
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
|
||||
if (NULL == tmp_data) {
|
||||
fprintf(stderr, "ex data del error: no such key:%s", key);
|
||||
return;
|
||||
}
|
||||
|
||||
rcu_hash_del(ex_data_rt->htable, key, key_len);
|
||||
}
|
||||
|
||||
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
|
||||
{
|
||||
void *ex_data = rcu_hash_find(ex_data_rt->htable, key, key_len);
|
||||
if (NULL == ex_data) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *dup_ex_data = NULL;
|
||||
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &ex_data,
|
||||
ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp);
|
||||
return dup_ex_data;
|
||||
}
|
||||
|
||||
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
return rcu_hash_count(ex_data_rt->htable);
|
||||
}
|
||||
|
||||
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array)
|
||||
{
|
||||
if (NULL == ex_data_rt->ex_schema) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return rcu_hash_list_updating_data(ex_data_rt->htable, ex_data_array);
|
||||
}
|
||||
|
||||
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
return rcu_hash_updating_flag(ex_data_rt->htable);
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
|
||||
#include <time.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#include <ctype.h>
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_utils.h"
|
||||
#include "utils.h"
|
||||
|
||||
#define MAAT_KV_MAX_KEY_LEN 512
|
||||
|
||||
|
||||
@@ -9,92 +9,179 @@
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <pthread.h>
|
||||
#include <limits.h>
|
||||
#include <string.h>
|
||||
#include <sys/prctl.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "maat_rule.h"
|
||||
#include "maat_config_monitor.h"
|
||||
#include "utils.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "maat_table_schema.h"
|
||||
|
||||
struct maat_runtime* create_maat_runtime(uint64_t version, struct maat *maat_instance)
|
||||
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
|
||||
{
|
||||
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
|
||||
|
||||
maat_rt->version = version;
|
||||
maat_rt->table_rt_mgr = maat_table_runtime_manager_create(maat_instance->table_mgr);
|
||||
maat_rt->max_table_num = maat_table_manager_get_size(maat_instance->table_mgr);
|
||||
maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr,
|
||||
maat_instance->nr_worker_thread,
|
||||
maat_instance->garbage_bin);
|
||||
maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr);
|
||||
maat_rt->max_thread_num = maat_instance->nr_worker_thread;
|
||||
|
||||
return maat_rt;
|
||||
}
|
||||
|
||||
void update_maat_runtime(struct maat_runtime *maat_rt)
|
||||
void maat_runtime_commit(struct maat_runtime *maat_rt)
|
||||
{
|
||||
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
|
||||
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, i);
|
||||
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
|
||||
if (NULL == table_rt) {
|
||||
continue;
|
||||
}
|
||||
|
||||
enum maat_table_type table_type = maat_table_runtime_get_type(table_rt);
|
||||
table_runtime_commit(table_rt, maat_rt->max_thread_num);
|
||||
}
|
||||
|
||||
maat_rt->last_update_time = time(NULL);
|
||||
}
|
||||
|
||||
void destroy_maat_runtime(struct maat_runtime *maat_rt)
|
||||
void maat_runtime_destroy(struct maat_runtime *maat_rt)
|
||||
{
|
||||
if (NULL == maat_rt) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (maat_rt->table_rt_mgr != NULL) {
|
||||
table_runtime_manager_destroy(maat_rt->table_rt_mgr);
|
||||
maat_rt->table_rt_mgr = NULL;
|
||||
}
|
||||
|
||||
free(maat_rt);
|
||||
}
|
||||
|
||||
void maat_start_cb(uint64_t new_version, int update_type, void *u_param)
|
||||
int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
|
||||
{
|
||||
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
|
||||
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
|
||||
if (NULL == table_rt) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int flag = table_runtime_updating_flag(table_rt);
|
||||
if (1 == flag) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void maat_start_cb(long long new_version, int update_type, void *u_param)
|
||||
{
|
||||
struct maat *maat_instance = (struct maat *)u_param;
|
||||
|
||||
if (update_type == CONFIG_UPDATE_TYPE_FULL) {
|
||||
maat_instance->rebuilding_maat_rt = create_maat_runtime(new_version, maat_instance);
|
||||
if (update_type == CM_UPDATE_TYPE_FULL) {
|
||||
maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance);
|
||||
} else {
|
||||
maat_instance->maat_version = new_version;
|
||||
}
|
||||
|
||||
table_schema_manager_all_plugin_cb_start(maat_instance->table_schema_mgr, update_type);
|
||||
}
|
||||
|
||||
int maat_update_cb(const char* table_name, const char* line, void *u_param)
|
||||
int maat_update_cb(const char *table_name, const char *line, void *u_param)
|
||||
{
|
||||
struct maat *maat_instance =(struct maat *)u_param;
|
||||
struct maat_runtime* maat_rt = NULL;
|
||||
int table_id = maat_table_manager_get_table_id(maat_instance->table_mgr, table_name);
|
||||
struct maat_table_schema* ptable = maat_table_manager_get_table_desc(maat_instance->table_mgr, table_id);
|
||||
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
|
||||
struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
|
||||
|
||||
if (maat_instance->rebuilding_maat_rt != NULL) {
|
||||
maat_rt = maat_instance->rebuilding_maat_rt;
|
||||
if (maat_instance->creating_maat_rt != NULL) {
|
||||
maat_rt = maat_instance->creating_maat_rt;
|
||||
} else {
|
||||
maat_rt = maat_instance->maat_rt;
|
||||
}
|
||||
|
||||
//TODO: update rule for table_schema
|
||||
update_maat_runtime();
|
||||
struct table_item *table_item = table_schema_line_to_item(line, table_schema);
|
||||
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
|
||||
table_runtime_update(table_rt, table_schema, line, table_item);
|
||||
free(table_item);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
|
||||
{
|
||||
uint32_t total = 0;
|
||||
struct table_runtime *table_rt = NULL;
|
||||
|
||||
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
|
||||
table_rt = table_runtime_get(maat_rt->table_rt_mgr, i);
|
||||
if (table_rt != NULL) {
|
||||
total += table_runtime_rule_count(table_rt);
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
void maat_finish_cb(void *u_param)
|
||||
{
|
||||
struct maat *maat_instance = (struct maat *)u_param;
|
||||
|
||||
if (maat_instance->latest_maat_rt != NULL) {
|
||||
update_maat_runtime(maat_instance->latest_maat_rt);
|
||||
table_schema_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr);
|
||||
|
||||
if (maat_instance->creating_maat_rt != NULL) {
|
||||
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
|
||||
maat_runtime_commit(maat_instance->creating_maat_rt);
|
||||
fprintf(stdout, "Full config version %llu load %d entries complete",
|
||||
maat_instance->creating_maat_rt->version,
|
||||
maat_instance->creating_maat_rt->rule_num);
|
||||
} else if (maat_instance->maat_rt != NULL) {
|
||||
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
|
||||
maat_instance->maat_rt->version = maat_instance->maat_version;
|
||||
maat_runtime_commit(maat_instance->maat_rt);
|
||||
fprintf(stdout, "Inc config version %llu load %d entries complete",
|
||||
maat_instance->maat_rt->version,
|
||||
maat_instance->maat_rt->rule_num);
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
}
|
||||
|
||||
void *rule_monitor_loop(void *arg)
|
||||
{
|
||||
char err_str[NAME_MAX] = {0};
|
||||
/* Defined by prctl: The name can be up to 16 bytes long, and should
|
||||
be null terminated if it contains fewer bytes. */
|
||||
char maat_name[16] = {0};
|
||||
struct maat *maat_instance = (struct maat *)arg;
|
||||
|
||||
if (strlen(maat_instance->instance_name) > 0) {
|
||||
snprintf(maat_name, sizeof(maat_name), "MAAT_%s", maat_instance->instance_name);
|
||||
} else {
|
||||
snprintf(maat_name, sizeof(maat_name), "MAAT");
|
||||
}
|
||||
|
||||
int ret = prctl(PR_SET_NAME, (unsigned long long)maat_name, NULL, NULL, NULL);
|
||||
assert(ret >= 0);
|
||||
|
||||
pthread_mutex_lock(&(maat_instance->background_update_mutex));
|
||||
/* if deferred load on */
|
||||
if (maat_instance->deferred_load != 0) {
|
||||
fprintf(stdout, "Deferred Loading ON, updating in %s.", __func__);
|
||||
maat_read_full_config(maat_instance);
|
||||
}
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
|
||||
while (maat_instance->is_running) {
|
||||
usleep(1000 * 1000);
|
||||
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
|
||||
if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
|
||||
switch (maat_instance->rule_import_type) {
|
||||
case RULE_IMPORT_TYPE_IRIS:
|
||||
switch (maat_instance->input_mode) {
|
||||
case DATA_SOURCE_IRIS_FILE:
|
||||
config_monitor_traverse(maat_instance->maat_version,
|
||||
maat_instance->iris_ctx.inc_dir,
|
||||
maat_start_cb,
|
||||
@@ -106,25 +193,38 @@ void *rule_monitor_loop(void *arg)
|
||||
break;
|
||||
}
|
||||
|
||||
if (maat_instance->latest_maat_rt != NULL) {
|
||||
if (maat_instance->creating_maat_rt != NULL) {
|
||||
struct maat_runtime *old_maat_rt = maat_instance->maat_rt;
|
||||
maat_instance->maat_rt = maat_instance->latest_maat_rt;
|
||||
maat_instance->maat_rt = maat_instance->creating_maat_rt;
|
||||
|
||||
if (old_maat_rt != NULL) {
|
||||
|
||||
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy);
|
||||
}
|
||||
|
||||
maat_instance->latest_maat_rt = NULL;
|
||||
maat_instance->creating_maat_rt = NULL;
|
||||
maat_instance->maat_version = maat_instance->maat_rt->version;
|
||||
maat_instance->last_full_version = maat_instance->maat_rt->version;
|
||||
}
|
||||
|
||||
if (maat_instance->maat_rt != NULL) {
|
||||
update_maat_runtime(maat_instance->maat_rt);
|
||||
int updating_flag = maat_runtime_updating_flag(maat_instance->maat_rt);
|
||||
time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time;
|
||||
|
||||
if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) {
|
||||
maat_runtime_commit(maat_instance->maat_rt);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
}
|
||||
|
||||
maat_garbage_collect_routine(maat_instance->garbage_bin);
|
||||
}
|
||||
|
||||
maat_runtime_destroy(maat_instance->maat_rt);
|
||||
maat_garbage_bin_free(maat_instance->garbage_bin);
|
||||
table_schema_manager_destroy(maat_instance->table_schema_mgr);
|
||||
free(maat_instance);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
@@ -8,55 +8,108 @@
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <assert.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <hs/hs.h>
|
||||
|
||||
#include "utils.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_ex_data.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "rcu_hash.h"
|
||||
#include "IPMatcher.h"
|
||||
|
||||
struct maat_expr_runtime {
|
||||
struct adapter_hs *hs;
|
||||
//rcu_hash *htable;
|
||||
#define MAAT_MAX_EXPR_ITEM_NUM 8
|
||||
|
||||
struct plugin_ex_data {
|
||||
void *data;
|
||||
void *user_data;
|
||||
};
|
||||
|
||||
struct maat_ip_plugin_runtime {
|
||||
//struct ip_matcher* ip_matcher;
|
||||
struct plugin_user_ctx {
|
||||
int table_id;
|
||||
struct ex_data_schema *ex_schema;
|
||||
};
|
||||
|
||||
struct expr_runtime {
|
||||
enum scan_mode scan_mode;
|
||||
struct adapter_hs *hs;
|
||||
struct adapter_hs_stream *hs_stream;
|
||||
struct rcu_hash_table *htable;
|
||||
};
|
||||
|
||||
struct ip_runtime {
|
||||
struct ip_matcher *ip_matcher;
|
||||
struct rcu_hash_table *htable;
|
||||
};
|
||||
|
||||
struct plugin_runtime {
|
||||
uint64_t acc_line_num;
|
||||
struct ex_data_runtime *ex_data_rt;
|
||||
};
|
||||
|
||||
struct ip_plugin_runtime {
|
||||
struct ip_matcher* ip_matcher;
|
||||
struct ex_data_runtime* ex_data_rt;
|
||||
};
|
||||
|
||||
struct maat_table_runtime {
|
||||
enum maat_table_type table_type; // table schema已有type??
|
||||
struct table_runtime {
|
||||
uint32_t rule_num;
|
||||
|
||||
uint32_t updating_rule_num;
|
||||
enum table_type table_type;
|
||||
union {
|
||||
struct maat_expr_runtime *expr_rt;
|
||||
struct maat_ip_plugin_runtime *ip_plugin_rt;
|
||||
struct expr_runtime expr_rt;
|
||||
struct ip_runtime ip_rt;
|
||||
struct plugin_runtime plugin_rt;
|
||||
struct ip_plugin_runtime ip_plugin_rt;
|
||||
};
|
||||
|
||||
struct maat_garbage_bin *ref_garbage_bin;
|
||||
//ex_data_rt
|
||||
//table相关指针
|
||||
};
|
||||
|
||||
struct maat_table_runtime_manager {
|
||||
struct maat_table_runtime **table_rt;
|
||||
struct table_runtime_manager {
|
||||
struct table_runtime **table_rt;
|
||||
size_t n_table_rt;
|
||||
|
||||
struct maat_garbage_bin *garbage_bin;
|
||||
};
|
||||
|
||||
struct maat_table_runtime *table_runtime_new(enum maat_table_type table_type, int max_thread_num, struct maat_garbage_bin* bin)
|
||||
void plugin_ex_data_free(void *user_ctx, void *data)
|
||||
{
|
||||
struct maat_table_runtime *table_rt = ALLOC(struct maat_table_runtime, 1);
|
||||
table_rt->table_type = table_type;
|
||||
switch (table_type) {
|
||||
struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx;
|
||||
long argl = ctx->ex_schema->argl;
|
||||
void *argp = ctx->ex_schema->argp;
|
||||
ctx->ex_schema->free_func(ctx->table_id, &data, argl, argp);
|
||||
}
|
||||
|
||||
void expr_ex_data_free(void *user_ctx, void *data)
|
||||
{
|
||||
free(data);
|
||||
}
|
||||
|
||||
struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin)
|
||||
{
|
||||
int table_id = table_schema_get_table_id(table_schema);
|
||||
struct table_runtime *table_rt = ALLOC(struct table_runtime, 1);
|
||||
table_rt->ref_garbage_bin = bin;
|
||||
table_rt->table_type = table_schema_get_table_type(table_schema);
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
table_rt->expr_rt->ex_data_rt = ex_data_runtime_new(NULL);
|
||||
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
|
||||
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
|
||||
break;
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
table_rt->ip_plugin_rt->ex_data_rt = ex_data_runtime_new(NULL);
|
||||
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
@@ -65,15 +118,20 @@ struct maat_table_runtime *table_runtime_new(enum maat_table_type table_type, in
|
||||
return table_rt;
|
||||
}
|
||||
|
||||
void table_runtime_free(struct maat_table_runtime * table_rt)
|
||||
void table_runtime_free(struct table_runtime * table_rt)
|
||||
{
|
||||
switch (table_rt->table_type)
|
||||
{
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
ex_data_runtime_free(table_rt->expr_rt->ex_data_rt);
|
||||
adapter_hs_destroy(table_rt->expr_rt.hs);
|
||||
rcu_hash_free(table_rt->expr_rt.htable);
|
||||
break;
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ex_data_runtime_free(table_rt->ip_plugin_rt->ex_data_rt);
|
||||
ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher);
|
||||
ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -81,28 +139,30 @@ void table_runtime_free(struct maat_table_runtime * table_rt)
|
||||
free(table_rt);
|
||||
}
|
||||
|
||||
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* garbage_bin)
|
||||
struct table_runtime_manager *
|
||||
table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
|
||||
struct maat_garbage_bin* garbage_bin)
|
||||
{
|
||||
if (NULL == table_mgr) {
|
||||
if (NULL == table_schema_mgr) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct maat_table_runtime_manager *table_rt_mgr = ALLOC(struct maat_table_runtime_manager, 1);
|
||||
table_rt_mgr->n_table_rt = maat_table_manager_get_size(table_mgr);
|
||||
table_rt_mgr->table_rt = ALLOC(struct maat_table_runtime *, table_rt_mgr->n_table_rt);
|
||||
struct table_runtime_manager *table_rt_mgr = ALLOC(struct table_runtime_manager, 1);
|
||||
table_rt_mgr->n_table_rt = table_schema_manager_get_size(table_schema_mgr);
|
||||
table_rt_mgr->table_rt = ALLOC(struct table_runtime *, table_rt_mgr->n_table_rt);
|
||||
|
||||
for (size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
|
||||
enum maat_table_type table_type = maat_table_manager_get_table_type(table_mgr, i);
|
||||
if (table_type == TABLE_TYPE_MAX) {
|
||||
struct table_schema *table_schema = table_schema_get(table_schema_mgr, i);
|
||||
if (NULL == table_schema) {
|
||||
continue;
|
||||
}
|
||||
table_rt_mgr->table_rt[i] = table_runtime_new(table_type, max_thread_num, garbage_bin);
|
||||
table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin);
|
||||
}
|
||||
|
||||
return table_rt_mgr;
|
||||
}
|
||||
|
||||
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr)
|
||||
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr)
|
||||
{
|
||||
for(size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
|
||||
table_runtime_free(table_rt_mgr->table_rt[i]);
|
||||
@@ -114,13 +174,462 @@ void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table
|
||||
free(table_rt_mgr);
|
||||
}
|
||||
|
||||
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id)
|
||||
struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id)
|
||||
{
|
||||
assert(table_id < (int)table_rt_mgr->n_table_rt);
|
||||
return table_rt_mgr->table_rt[table_id];
|
||||
}
|
||||
|
||||
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt)
|
||||
size_t table_runtime_rule_count(struct table_runtime *table_rt)
|
||||
{
|
||||
return table_rt->rule_num;
|
||||
}
|
||||
|
||||
enum table_type table_runtime_get_type(struct table_runtime* table_rt)
|
||||
{
|
||||
return table_rt->table_type;
|
||||
}
|
||||
|
||||
int table_runtime_scan_string(struct table_runtime* table_rt, int thread_id, const char *data, size_t data_len,
|
||||
int result[], size_t *n_result)
|
||||
{
|
||||
return adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, result, n_result);
|
||||
}
|
||||
|
||||
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id)
|
||||
{
|
||||
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id);
|
||||
table_rt->expr_rt.hs_stream = hs_stream;
|
||||
}
|
||||
|
||||
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len,
|
||||
int result[], size_t *n_result)
|
||||
{
|
||||
return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result);
|
||||
}
|
||||
|
||||
void table_runtime_stream_close(struct table_runtime *table_rt)
|
||||
{
|
||||
adapter_hs_stream_close(table_rt->expr_rt.hs_stream);
|
||||
table_rt->expr_rt.hs_stream = NULL;
|
||||
}
|
||||
|
||||
struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item)
|
||||
{
|
||||
size_t i = 0;
|
||||
size_t sub_expr_cnt = 0;
|
||||
char *pos = NULL;
|
||||
char *saveptr = NULL;
|
||||
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
|
||||
and_expr_t *expr_rule = ALLOC(and_expr_t, 1);
|
||||
|
||||
switch (expr_item->expr_type) {
|
||||
case EXPR_TYPE_AND:
|
||||
case EXPR_TYPE_REGEX:
|
||||
for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) {
|
||||
char *tmp = strtok_r_esc(pos, '&', &saveptr);
|
||||
if (NULL == tmp) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
|
||||
fprintf(stderr, "item_id:%d too many patterns", expr_item->item_id);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
sub_key_array[i] = tmp;
|
||||
if (expr_item->expr_type == EXPR_TYPE_REGEX) {
|
||||
sub_key_array[i] = str_unescape_and(sub_key_array[i]);
|
||||
} else {
|
||||
sub_key_array[i] = str_unescape(sub_key_array[i]);
|
||||
}
|
||||
}
|
||||
sub_expr_cnt = i;
|
||||
break;
|
||||
case EXPR_TYPE_STRING:
|
||||
sub_expr_cnt = 1;
|
||||
sub_key_array[0] = expr_item->keywords;
|
||||
sub_key_array[0] = str_unescape(sub_key_array[0]);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
for (i = 0; i < sub_expr_cnt; i++) {
|
||||
expr_rule->expr_id = expr_item->item_id;
|
||||
expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i]));
|
||||
memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i]));
|
||||
expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
|
||||
expr_rule->patterns[i].type = expr_item->expr_type;
|
||||
}
|
||||
expr_rule->n_patterns = sub_expr_cnt;
|
||||
|
||||
return expr_rule;
|
||||
}
|
||||
|
||||
void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
|
||||
and_expr_t *expr_rule, int is_valid)
|
||||
{
|
||||
void *data = NULL;
|
||||
|
||||
if (is_valid == 0) {
|
||||
//delete
|
||||
data = rcu_hash_find(expr_rt->htable, key, key_len);
|
||||
if (NULL == data) {
|
||||
//log_error("the key:%s not exist, so can't be deleted.");
|
||||
return;
|
||||
}
|
||||
rcu_hash_del(expr_rt->htable, key, key_len);
|
||||
} else {
|
||||
//add
|
||||
data = rcu_hash_find(expr_rt->htable, key, key_len);
|
||||
if (data != NULL) {
|
||||
//log_error("the key:%s already exist, so can't be added.");
|
||||
return;
|
||||
}
|
||||
and_expr_t *data = ALLOC(and_expr_t, 1);
|
||||
memcpy(data, expr_rule, sizeof(and_expr_t));
|
||||
rcu_hash_add(expr_rt->htable, key, key_len, (void *)data);
|
||||
}
|
||||
}
|
||||
|
||||
void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema,
|
||||
const char *row, char *key, size_t key_len, int is_valid)
|
||||
{
|
||||
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1);
|
||||
ex_data->data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
|
||||
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
|
||||
size_t cb_count = plugin_table_schema_callback_count(table_schema);
|
||||
|
||||
/* already set plugin_table_schema's ex_data_schema */
|
||||
if (1 == set_flag) {
|
||||
if (is_valid == 0) {
|
||||
// delete
|
||||
ex_data_runtime_del_ex_data(plugin_rt->ex_data_rt, key, key_len);
|
||||
} else {
|
||||
// add
|
||||
ex_data_runtime_add_ex_data(plugin_rt->ex_data_rt, key, key_len, (void *)ex_data);
|
||||
}
|
||||
}
|
||||
|
||||
/* plugin table schema has callback */
|
||||
if (cb_count > 0) {
|
||||
plugin_table_schema_all_cb_update(table_schema, row);
|
||||
}
|
||||
|
||||
if ((0 == set_flag) && (0 == cb_count)) {
|
||||
ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
|
||||
}
|
||||
|
||||
plugin_rt->acc_line_num++;
|
||||
}
|
||||
|
||||
void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema,
|
||||
const char *row, char *key, size_t key_len,
|
||||
struct ip_rule *ip_rule, int is_valid)
|
||||
{
|
||||
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1);
|
||||
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
|
||||
ex_data->data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
|
||||
ex_data->user_data = ip_rule;
|
||||
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
|
||||
|
||||
if (1 == set_flag) {
|
||||
if (0 == is_valid) {
|
||||
//delete
|
||||
ex_data_runtime_del_ex_data(ex_data_rt, key, key_len);
|
||||
} else {
|
||||
//add
|
||||
ex_data_runtime_add_ex_data(ip_plugin_rt->ex_data_rt, key, key_len, ex_data);
|
||||
}
|
||||
} else {
|
||||
ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *row, struct table_item *table_item)
|
||||
{
|
||||
int is_valid = -1;
|
||||
char *key = NULL;
|
||||
size_t key_len = 0;
|
||||
and_expr_t *expr_rule = NULL;
|
||||
struct ip_rule *ip_rule = NULL;
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
is_valid = table_item->expr_item.is_valid;
|
||||
expr_rule = expr_item_to_expr_rule(&table_item->expr_item);
|
||||
key = (char *)&(table_item->expr_item.item_id);
|
||||
expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid);
|
||||
free(expr_rule);
|
||||
break;
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
is_valid = table_item->plugin_item.is_valid;
|
||||
key = table_item->plugin_item.key;
|
||||
key_len = table_item->plugin_item.key_len;
|
||||
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
is_valid = table_item->ip_plugin_item.is_valid;
|
||||
ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item);
|
||||
key = (char *)&(table_item->ip_plugin_item.item_id);
|
||||
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid);
|
||||
free(ip_rule);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (is_valid == 0) {
|
||||
table_rt->rule_num--;
|
||||
} else {
|
||||
table_rt->rule_num++;
|
||||
}
|
||||
}
|
||||
|
||||
int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
|
||||
{
|
||||
struct expr_runtime *expr_rt = &(table_rt->expr_rt);
|
||||
void **ex_data_array = NULL;
|
||||
and_expr_t *rules = NULL;
|
||||
size_t rule_cnt = 0;
|
||||
int ret = 0;
|
||||
|
||||
rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array);
|
||||
rules = ALLOC(and_expr_t, rule_cnt);
|
||||
|
||||
for (size_t i = 0; i < rule_cnt; i++) {
|
||||
rules[i] = *(and_expr_t *)ex_data_array[i];
|
||||
}
|
||||
|
||||
struct adapter_hs *new_adapter_hs = NULL;
|
||||
struct adapter_hs *old_adapter_hs = NULL;
|
||||
if (rule_cnt > 0) {
|
||||
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt);
|
||||
if (NULL == new_adapter_hs) {
|
||||
ret = -1;
|
||||
}
|
||||
}
|
||||
|
||||
old_adapter_hs = expr_rt->hs;
|
||||
expr_rt->hs = new_adapter_hs;
|
||||
|
||||
maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy);
|
||||
rcu_hash_commit(expr_rt->htable);
|
||||
table_rt->rule_num = rcu_hash_count(expr_rt->htable);
|
||||
|
||||
free(rules);
|
||||
free(ex_data_array);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int plugin_runtime_commit(struct table_runtime *table_rt)
|
||||
{
|
||||
ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt);
|
||||
table_rt->rule_num = ex_data_runtime_ex_data_count(table_rt->plugin_rt.ex_data_rt);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ip_plugin_runtime_commit(struct table_runtime *table_rt)
|
||||
{
|
||||
struct plugin_ex_data **ex_data_array = NULL;
|
||||
struct ip_rule *rules = NULL;
|
||||
size_t rule_cnt = 0;
|
||||
int ret = 0;
|
||||
struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt);
|
||||
|
||||
rule_cnt = ex_data_runtime_list_updating_ex_data(ip_plugin_rt->ex_data_rt, (void ***)&ex_data_array);
|
||||
rules = ALLOC(struct ip_rule, rule_cnt);
|
||||
|
||||
for (size_t i = 0; i < rule_cnt; i++) {
|
||||
rules[i] = *(struct ip_rule *)ex_data_array[i]->user_data;
|
||||
}
|
||||
|
||||
struct ip_matcher *new_ip_matcher = NULL;
|
||||
struct ip_matcher *old_ip_matcher = NULL;
|
||||
size_t mem_used = 0;
|
||||
|
||||
if (rule_cnt > 0) {
|
||||
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
|
||||
if (NULL == new_ip_matcher) {
|
||||
ret = -1;
|
||||
}
|
||||
}
|
||||
|
||||
old_ip_matcher = ip_plugin_rt->ip_matcher;
|
||||
ip_plugin_rt->ip_matcher = new_ip_matcher;
|
||||
maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
|
||||
ex_data_runtime_commit(ip_plugin_rt->ex_data_rt);
|
||||
table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt);
|
||||
|
||||
free(rules);
|
||||
free(ex_data_array);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int expr_runtime_updating_flag(struct expr_runtime *expr_rt)
|
||||
{
|
||||
return rcu_hash_updating_flag(expr_rt->htable);
|
||||
}
|
||||
|
||||
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt)
|
||||
{
|
||||
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
|
||||
}
|
||||
|
||||
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt)
|
||||
{
|
||||
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
|
||||
}
|
||||
|
||||
int table_runtime_updating_flag(struct table_runtime *table_rt)
|
||||
{
|
||||
int updating_flag = 0;
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
updating_flag = expr_runtime_updating_flag(&(table_rt->expr_rt));
|
||||
break;
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
updating_flag = plugin_runtime_updating_flag(&(table_rt->plugin_rt));
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
updating_flag = ip_plugin_runtime_updating_flag(&(table_rt->ip_plugin_rt));
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return updating_flag;
|
||||
}
|
||||
|
||||
void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
|
||||
{
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
expr_runtime_commit(table_rt, nr_worker_thread);
|
||||
break;
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
plugin_runtime_commit(table_rt);
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ip_plugin_runtime_commit(table_rt);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
size_t plugin_table_runtime_cached_row_count(struct table_runtime* table_rt)
|
||||
{
|
||||
size_t row_count = 0;
|
||||
struct ex_data_runtime *ex_data_rt = NULL;
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
row_count = ex_data_runtime_cached_row_count(ex_data_rt);
|
||||
|
||||
return row_count;
|
||||
}
|
||||
|
||||
const char* plugin_table_runtime_get_cached_row(struct table_runtime* table_rt, size_t row_seq)
|
||||
{
|
||||
const char *line = NULL;
|
||||
struct ex_data_runtime *ex_data_rt = NULL;
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
line = ex_data_runtime_cached_row_get(ex_data_rt, row_seq);
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema,
|
||||
const char *key, size_t key_len)
|
||||
{
|
||||
void *ex_data = NULL;
|
||||
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
|
||||
if (0 == set_flag) {
|
||||
assert(0);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
enum table_type table_type = table_schema_get_table_type(table_schema);
|
||||
switch (table_type) {
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
ex_data = ex_data_runtime_get_ex_data(table_rt->plugin_rt.ex_data_rt, key, key_len);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return ex_data;
|
||||
}
|
||||
|
||||
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt)
|
||||
{
|
||||
struct ex_data_runtime *ex_data_rt = NULL;
|
||||
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
ex_data_rt = table_rt->plugin_rt.ex_data_rt;
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return ex_data_rt;
|
||||
}
|
||||
|
||||
void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema)
|
||||
{
|
||||
struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema);
|
||||
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
|
||||
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
|
||||
struct plugin_user_ctx *user_ctx = ALLOC(struct plugin_user_ctx, 1);
|
||||
|
||||
user_ctx->table_id = table_schema_get_table_id(table_schema);
|
||||
user_ctx->ex_schema = ex_data_schema;
|
||||
ex_data_runtime_set_user_ctx(ex_data_rt, user_ctx);
|
||||
|
||||
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
|
||||
for (size_t i = 0; i < n_cached_row; i++) {
|
||||
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
|
||||
switch (table_rt->table_type) {
|
||||
case TABLE_TYPE_PLUGIN:
|
||||
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
ex_data_runtime_clear_row_cache(ex_data_rt);
|
||||
|
||||
table_runtime_commit(table_rt, 0);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,28 +8,174 @@
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
|
||||
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len)
|
||||
char *maat_strdup(const char *s)
|
||||
{
|
||||
const char* seps=" \t";
|
||||
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
|
||||
sds dup_line = sdsdup(line);
|
||||
int i=0, ret=-1;
|
||||
for (str = dup_line; ; str = NULL)
|
||||
{
|
||||
if (NULL == s) {
|
||||
return NULL;
|
||||
}
|
||||
char *d = (char *)malloc(strlen(s) + 1);
|
||||
memcpy(d, s, strlen(s) + 1);
|
||||
|
||||
return d;
|
||||
}
|
||||
|
||||
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)
|
||||
{
|
||||
const char *seps=" \t";
|
||||
char *saveptr=NULL, *subtoken=NULL, *str=NULL;
|
||||
char *dup_line = maat_strdup(line);
|
||||
int i = 0, ret = -1;
|
||||
for (str = dup_line; ; str = NULL) {
|
||||
subtoken = strtok_r(str, seps, &saveptr);
|
||||
if (subtoken == NULL)
|
||||
break;
|
||||
if(i==column_seq-1)
|
||||
{
|
||||
*offset=subtoken-dup_line;
|
||||
*len=strlen(subtoken);
|
||||
ret=0;
|
||||
if (i == column_seq - 1) {
|
||||
*offset = subtoken - dup_line;
|
||||
*len = strlen(subtoken);
|
||||
ret = 0;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
free(dup_line);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz)
|
||||
{
|
||||
int ret = 0;
|
||||
FILE *fp = NULL;
|
||||
struct stat fstat_buf;
|
||||
size_t read_size=0;
|
||||
|
||||
ret = stat(file_name, &fstat_buf);
|
||||
if (ret != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
fp = fopen(file_name, "r");
|
||||
if (fp == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*out_sz = fstat_buf.st_size;
|
||||
*pp_out = (unsigned char *)calloc(1, *out_sz+1);
|
||||
read_size = fread(*pp_out, 1, *out_sz, fp);
|
||||
if (read_size != *out_sz) {
|
||||
free(*pp_out);
|
||||
pp_out = NULL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
fclose(fp);
|
||||
fp = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static char *strchr_esc(char *s, const char delim)
|
||||
{
|
||||
char *token = NULL;
|
||||
|
||||
if (NULL == s) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (token = s; *token != '\0'; token++) {
|
||||
if (*token == '\\') {
|
||||
token++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if(*token == delim) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (*token == '\0') {
|
||||
return NULL;
|
||||
} else {
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
char *strtok_r_esc(char *s, const char delim, char **save_ptr)
|
||||
{
|
||||
char *token = NULL;
|
||||
|
||||
if (NULL == s) {
|
||||
s = *save_ptr;
|
||||
}
|
||||
|
||||
/* Scan leading delimiters. */
|
||||
token = strchr_esc(s,delim);
|
||||
if (NULL == token) {
|
||||
*save_ptr = token;
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Find the end of the token. */
|
||||
*token = '\0';
|
||||
token++;
|
||||
*save_ptr = token;
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
char *str_unescape_and(char *s)
|
||||
{
|
||||
size_t i = 0;
|
||||
size_t j = 0;
|
||||
|
||||
for (i = 0,j = 0; i < strlen(s); i++) {
|
||||
if (s[i] == '\\' && s[i+1] == '&') {
|
||||
s[j] = '&';
|
||||
i++;
|
||||
j++;
|
||||
} else {
|
||||
s[j] = s[i];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
s[j] = '\0';
|
||||
return s;
|
||||
}
|
||||
|
||||
char *str_unescape(char *s)
|
||||
{
|
||||
size_t i=0;
|
||||
size_t j=0;
|
||||
|
||||
for (i = 0, j = 0; i < strlen(s); i++) {
|
||||
if (s[i] == '\\') {
|
||||
switch (s[i+1]) {
|
||||
case '&':
|
||||
s[j] = '&';
|
||||
break;
|
||||
case 'b':
|
||||
s[j] = ' ';//space,0x20;
|
||||
break;
|
||||
case '\\':
|
||||
s[j] = '\\';
|
||||
break;
|
||||
default:
|
||||
s[j] = s[i];
|
||||
i--; //undo the followed i++
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
j++;
|
||||
} else {
|
||||
s[j] = s[i];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
s[j] = '\0';
|
||||
return s;
|
||||
}
|
||||
170
src/rcu_hash.cpp
170
src/rcu_hash.cpp
@@ -11,12 +11,17 @@
|
||||
#include <assert.h>
|
||||
#include <pthread.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/queue.h>
|
||||
|
||||
#include "rcu_hash.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_garbage_collection.h"
|
||||
#include "utils.h"
|
||||
|
||||
#define GARBAGE_DEFAULT_TIMEOUT 60
|
||||
struct rcu_hash_garbage_bag {
|
||||
void *garbage;
|
||||
void (* garbage_free)(void *garbage);
|
||||
TAILQ_ENTRY(rcu_hash_garbage_bag) entries;
|
||||
};
|
||||
TAILQ_HEAD(rcu_hash_garbage_q, rcu_hash_garbage_bag);
|
||||
|
||||
struct rcu_hash_table {
|
||||
int is_updating;
|
||||
@@ -26,8 +31,11 @@ struct rcu_hash_table {
|
||||
struct rcu_hash_node *hashmap_a;
|
||||
struct rcu_hash_node *hashmap_b;
|
||||
|
||||
void (* data_free)(void *data);
|
||||
struct maat_garbage_bin *garbage_bin;
|
||||
void (*data_free_fn)(void *user_ctx, void *data);
|
||||
void *user_ctx;
|
||||
|
||||
struct rcu_hash_garbage_q garbage_q;
|
||||
size_t garbage_q_len;
|
||||
|
||||
pthread_mutex_t update_mutex;
|
||||
};
|
||||
@@ -37,26 +45,57 @@ struct rcu_hash_node {
|
||||
size_t key_len;
|
||||
void *data; //table_runtime解析成两个成员
|
||||
|
||||
/* htable the node belongs to */
|
||||
struct rcu_hash_table *htable;
|
||||
|
||||
UT_hash_handle hh_a;
|
||||
UT_hash_handle hh_b;
|
||||
};
|
||||
|
||||
void rcu_hash_node_free(struct rcu_hash_node *node, void (* data_free)(void *data))
|
||||
void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q* garbage_q)
|
||||
{
|
||||
struct rcu_hash_garbage_bag *p = NULL;
|
||||
|
||||
while ((p = TAILQ_FIRST(garbage_q)) != NULL) {
|
||||
p->garbage_free(p->garbage);
|
||||
TAILQ_REMOVE(garbage_q, p, entries);
|
||||
free(p);
|
||||
}
|
||||
}
|
||||
|
||||
size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable)
|
||||
{
|
||||
return htable->garbage_q_len;
|
||||
}
|
||||
|
||||
void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, void* garbage, void (* func)(void *))
|
||||
{
|
||||
struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1);
|
||||
|
||||
bag->garbage = garbage;
|
||||
bag->garbage_free = func;
|
||||
TAILQ_INSERT_TAIL(garbage_q, bag, entries);
|
||||
}
|
||||
|
||||
void rcu_hash_node_free(struct rcu_hash_node *node)
|
||||
{
|
||||
if (NULL == node) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (node->key != NULL) {
|
||||
free(node->key);
|
||||
}
|
||||
|
||||
if (node->data != NULL) {
|
||||
data_free(node->data);
|
||||
node->htable->data_free_fn(node->htable->user_ctx, node->data);
|
||||
}
|
||||
|
||||
free(node);
|
||||
}
|
||||
|
||||
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
|
||||
struct rcu_hash_table *rcu_hash_new(rcu_hash_data_free_fn *free_fn)
|
||||
{
|
||||
if (NULL == data_free) {
|
||||
if (NULL == free_fn) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -64,8 +103,9 @@ struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
|
||||
|
||||
htable->is_updating = 0;
|
||||
htable->effective_hash = 'a';
|
||||
htable->garbage_bin = maat_garbage_bin_new(GARBAGE_DEFAULT_TIMEOUT);
|
||||
htable->data_free = data_free;
|
||||
TAILQ_INIT(&htable->garbage_q);
|
||||
htable->garbage_q_len = 0;
|
||||
htable->data_free_fn = free_fn;
|
||||
pthread_mutex_init(&htable->update_mutex, NULL);
|
||||
|
||||
return htable;
|
||||
@@ -73,28 +113,36 @@ struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
|
||||
|
||||
void rcu_hash_free(struct rcu_hash_table *htable)
|
||||
{
|
||||
if (NULL == htable) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
struct rcu_hash_node *item = NULL;
|
||||
|
||||
if (htable != NULL) {
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_ITER(hh_a, htable->hashmap_a, item, tmp) {
|
||||
HASH_DELETE(hh_a, htable->hashmap_a, item);
|
||||
rcu_hash_node_free(item, htable->data_free);
|
||||
rcu_hash_node_free(item);
|
||||
}
|
||||
|
||||
} else {
|
||||
HASH_ITER(hh_b, htable->hashmap_b, item, tmp) {
|
||||
HASH_DELETE(hh_b, htable->hashmap_b, item);
|
||||
rcu_hash_node_free(item, htable->data_free);
|
||||
rcu_hash_node_free(item);
|
||||
}
|
||||
}
|
||||
|
||||
maat_garbage_bin_free(htable->garbage_bin);
|
||||
rcu_hash_garbage_queue_free(&(htable->garbage_q));
|
||||
pthread_mutex_destroy(&htable->update_mutex);
|
||||
|
||||
free(htable);
|
||||
}
|
||||
|
||||
void rcu_hash_update_prepare(struct rcu_hash_table *htable)
|
||||
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx)
|
||||
{
|
||||
htable->user_ctx = user_ctx;
|
||||
}
|
||||
|
||||
void rcu_hash_commit_prepare(struct rcu_hash_table *htable)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
@@ -116,23 +164,31 @@ void rcu_hash_update_prepare(struct rcu_hash_table *htable)
|
||||
|
||||
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data)
|
||||
{
|
||||
if (NULL == htable || NULL == key || 0 == key_len) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1);
|
||||
|
||||
node->key = (char *)malloc(sizeof(char) * key_len);
|
||||
memcpy(node->key, key, key_len);
|
||||
node->key_len = key_len;
|
||||
node->data = data;
|
||||
node->htable = htable;
|
||||
|
||||
if (!htable->is_updating) {
|
||||
rcu_hash_update_prepare(htable);
|
||||
rcu_hash_commit_prepare(htable);
|
||||
}
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
if (NULL == node) {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, tmp);
|
||||
if (NULL == tmp) {
|
||||
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
}
|
||||
} else {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
if (NULL == node) {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, tmp);
|
||||
if (NULL == tmp) {
|
||||
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
}
|
||||
}
|
||||
@@ -140,12 +196,15 @@ void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len
|
||||
|
||||
void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
|
||||
if (!htable->is_updating) {
|
||||
rcu_hash_update_prepare(htable);
|
||||
if (NULL == htable || NULL == key || 0 == key_len) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!htable->is_updating) {
|
||||
rcu_hash_commit_prepare(htable);
|
||||
}
|
||||
|
||||
struct rcu_hash_node *node = NULL;
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
@@ -159,14 +218,18 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len
|
||||
}
|
||||
|
||||
if (node != NULL) {
|
||||
maat_garbage_bagging(htable->garbage_bin, node, (void (*)(void*))rcu_hash_node_free);
|
||||
rcu_hash_garbage_bagging(&(htable->garbage_q), node, (void (*)(void*))rcu_hash_node_free);
|
||||
htable->garbage_q_len++;
|
||||
}
|
||||
}
|
||||
|
||||
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
if (NULL == htable || NULL == key || 0 == key_len) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct rcu_hash_node *node = NULL;
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
@@ -182,8 +245,12 @@ void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_l
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t rcu_hash_counts(struct rcu_hash_table *htable)
|
||||
size_t rcu_hash_count(struct rcu_hash_table *htable)
|
||||
{
|
||||
if (NULL == htable) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
return HASH_CNT(hh_a, htable->hashmap_a);
|
||||
} else {
|
||||
@@ -193,6 +260,10 @@ size_t rcu_hash_counts(struct rcu_hash_table *htable)
|
||||
|
||||
void rcu_hash_commit(struct rcu_hash_table *htable)
|
||||
{
|
||||
if (NULL == htable) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!htable->is_updating) {
|
||||
return;
|
||||
}
|
||||
@@ -221,7 +292,40 @@ void rcu_hash_commit(struct rcu_hash_table *htable)
|
||||
}
|
||||
}
|
||||
htable->is_updating = 0;
|
||||
//maat_garbage_collect_by_force(htable->garbage_bin);
|
||||
//rcu_garbage
|
||||
|
||||
rcu_hash_garbage_queue_free(&(htable->garbage_q));
|
||||
htable->garbage_q_len = 0;
|
||||
|
||||
pthread_mutex_unlock(&htable->update_mutex);
|
||||
}
|
||||
|
||||
size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array)
|
||||
{
|
||||
size_t i = 0;
|
||||
size_t node_cnt = 0;
|
||||
struct rcu_hash_node *node = NULL, *tmp = NULL;
|
||||
|
||||
assert(htable->is_updating == 1);
|
||||
if (htable->effective_hash == 'a') {
|
||||
node_cnt = HASH_CNT(hh_b, htable->hashmap_b);
|
||||
*data_array = ALLOC(void *, node_cnt);
|
||||
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
|
||||
(*data_array)[i] = node->data;
|
||||
i++;
|
||||
}
|
||||
} else {
|
||||
node_cnt = HASH_CNT(hh_a, htable->hashmap_a);
|
||||
*data_array = ALLOC(void *, node_cnt);
|
||||
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
|
||||
(*data_array)[i] = node->data;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return node_cnt;
|
||||
}
|
||||
|
||||
int rcu_hash_updating_flag(struct rcu_hash_table *htable)
|
||||
{
|
||||
return htable->is_updating;
|
||||
}
|
||||
@@ -1,11 +1,16 @@
|
||||
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/deps)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/scanner)
|
||||
|
||||
add_executable(maat_api_gtest maat_api_gtest.cpp)
|
||||
target_link_libraries(maat_api_gtest maat_frame_static gtest_static)
|
||||
|
||||
add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
|
||||
target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
|
||||
|
||||
add_executable(rcu_hash_gtest rcu_hash_gtest.cpp)
|
||||
target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static)
|
||||
|
||||
add_executable(maat_framework_gtest maat_framework_gtest.cpp)
|
||||
target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
|
||||
|
||||
file(COPY rule DESTINATION ./)
|
||||
file(COPY table_info.conf DESTINATION ./)
|
||||
file(COPY and_expr.conf DESTINATION ./)
|
||||
@@ -1,6 +1,6 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "maat/maat.h"
|
||||
|
||||
#include "../include/maat/maat.h"
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
TEST(EQ_Test, Always_True) {
|
||||
EXPECT_EQ(1, 1);
|
||||
|
||||
41
test/maat_framework_gtest.cpp
Normal file
41
test/maat_framework_gtest.cpp
Normal file
@@ -0,0 +1,41 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "maat/maat.h"
|
||||
#include "maat_rule.h"
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_table_runtime.h"
|
||||
|
||||
struct maat *g_maat_instance = NULL;
|
||||
const char *table_info_path = "/home/liuwentan/project/maat-v4/test/table_info.conf";
|
||||
const char *rule_path = "/home/liuwentan/project/maat-v4/test/rule/full/index";
|
||||
|
||||
TEST(maat_scan_string, literal) {
|
||||
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
|
||||
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
|
||||
char data[64] = "www.baidu.com";
|
||||
int result_array[5] = {0};
|
||||
size_t n_result_array = 0;
|
||||
int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result_array, 1);
|
||||
EXPECT_EQ(result_array[0], 101);
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
int ret=0;
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
struct maat_options *opts = maat_options_new();
|
||||
maat_options_set_iris_full_dir(opts, rule_path);
|
||||
|
||||
g_maat_instance = maat_new(opts, table_info_path);
|
||||
EXPECT_NE(g_maat_instance, nullptr);
|
||||
|
||||
ret=RUN_ALL_TESTS();
|
||||
|
||||
maat_free(g_maat_instance);
|
||||
g_maat_instance = NULL;
|
||||
|
||||
return ret;
|
||||
}
|
||||
@@ -1,16 +1,16 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "../src/inc_internal/rcu_hash.h"
|
||||
#include "../include/utils.h"
|
||||
|
||||
#include "rcu_hash.h"
|
||||
#include "maat_utils.h"
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
struct user_data {
|
||||
int id;
|
||||
char name[32];
|
||||
};
|
||||
|
||||
void data_free(void *data)
|
||||
void data_free(void *user_ctx, void *data)
|
||||
{
|
||||
|
||||
free(data);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_new, invalid_input_parameter) {
|
||||
@@ -18,27 +18,236 @@ TEST(rcu_hash_new, invalid_input_parameter) {
|
||||
EXPECT_EQ(htable, nullptr);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_add, one) {
|
||||
TEST(rcu_hash_add_one_node, single_thread) {
|
||||
/* add one node to hash */
|
||||
struct rcu_hash_table *htable = rcu_hash_new(data_free);
|
||||
EXPECT_NE(htable, nullptr);
|
||||
|
||||
struct user_data *data = ALLOC(struct user_data, 1);
|
||||
data->id = 101;
|
||||
char *name = "www.baidu.com";
|
||||
char name[64] = "www.baidu.com";
|
||||
memcpy(data->name, name, strlen(name));
|
||||
char *key = "http_url";
|
||||
char key[64] = "http_url";
|
||||
size_t key_len = strlen(key);
|
||||
|
||||
/* add to hash */
|
||||
rcu_hash_add(htable, key, key_len, (void *)data);
|
||||
|
||||
/* find in hash before commit */
|
||||
void *res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
int ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
/* find in hash after commit */
|
||||
res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
struct user_data *res_data = (struct user_data *)res;
|
||||
EXPECT_EQ(res_data->id, 101);
|
||||
EXPECT_STREQ(res_data->name, "www.baidu.com");
|
||||
|
||||
ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 1);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_free(htable);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_add_multi_node, single_thread) {
|
||||
/* add multi node to hash */
|
||||
struct rcu_hash_table *htable = rcu_hash_new(data_free);
|
||||
EXPECT_NE(htable, nullptr);
|
||||
|
||||
struct user_data *data0 = ALLOC(struct user_data, 1);
|
||||
data0->id = 101;
|
||||
char name0[64] = "www.baidu.com";
|
||||
memcpy(data0->name, name0, strlen(name0));
|
||||
char key0[64] = "http_url";
|
||||
size_t key0_len = strlen(key0);
|
||||
rcu_hash_add(htable, key0, key0_len, (void *)data0);
|
||||
|
||||
struct user_data *data1 = ALLOC(struct user_data, 1);
|
||||
data1->id = 102;
|
||||
char name1[64] = "127.0.0.1";
|
||||
memcpy(data1->name, name1, strlen(name1));
|
||||
char key1[64] = "http_host";
|
||||
size_t key1_len = strlen(key1);
|
||||
rcu_hash_add(htable, key1, key1_len, (void *)data1);
|
||||
|
||||
/* find in hash before commit */
|
||||
void *res = rcu_hash_find(htable, key0, key0_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
int ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
/* find in hash after commit */
|
||||
res = rcu_hash_find(htable, key0, key0_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
struct user_data *res_data0 = (struct user_data *)res;
|
||||
EXPECT_EQ(res_data0->id, 101);
|
||||
EXPECT_STREQ(res_data0->name, "www.baidu.com");
|
||||
|
||||
res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
struct user_data *res_data1 = (struct user_data *)res;
|
||||
EXPECT_EQ(res_data1->id, 102);
|
||||
EXPECT_STREQ(res_data1->name, "127.0.0.1");
|
||||
|
||||
ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 2);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_free(htable);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_del_one_node, single_thread) {
|
||||
/* case1: add and del before commit */
|
||||
struct rcu_hash_table *htable = rcu_hash_new(data_free);
|
||||
EXPECT_NE(htable, nullptr);
|
||||
|
||||
struct user_data *data = ALLOC(struct user_data, 1);
|
||||
data->id = 101;
|
||||
char name[64] = "www.baidu.com";
|
||||
memcpy(data->name, name, strlen(name));
|
||||
char key[64] = "http_url";
|
||||
size_t key_len = strlen(key);
|
||||
|
||||
/* add to hash */
|
||||
rcu_hash_add(htable, key, key_len, (void *)data);
|
||||
|
||||
/* find in hash before commit */
|
||||
void *res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
int ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_del(htable, key, key_len);
|
||||
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
/* find in hash after commit */
|
||||
res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
/* case2: add && commit, and del */
|
||||
struct user_data *data1 = ALLOC(struct user_data, 1);
|
||||
data1->id = 102;
|
||||
char name1[64] = "127.0.0.1";
|
||||
memcpy(data1->name, name1, strlen(name1));
|
||||
char key1[64] = "http_host";
|
||||
size_t key1_len = strlen(key1);
|
||||
rcu_hash_add(htable, key1, key1_len, (void *)data1);
|
||||
|
||||
/* add commit */
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
rcu_hash_del(htable, key1, key1_len);
|
||||
|
||||
res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
struct user_data *res_data = (struct user_data *)res;
|
||||
EXPECT_EQ(res_data->id, 102);
|
||||
EXPECT_STREQ(res_data->name, "127.0.0.1");
|
||||
|
||||
ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 1);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 1);
|
||||
|
||||
/* delete commit */
|
||||
rcu_hash_commit(htable);
|
||||
res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_free(htable);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_del_multi_node, single_thread) {
|
||||
/* case1: add and del before commit */
|
||||
struct rcu_hash_table *htable = rcu_hash_new(data_free);
|
||||
EXPECT_NE(htable, nullptr);
|
||||
|
||||
struct user_data *data1 = ALLOC(struct user_data, 1);
|
||||
data1->id = 101;
|
||||
char name1[64] = "www.baidu.com";
|
||||
memcpy(data1->name, name1, strlen(name1));
|
||||
char key1[64] = "http_url";
|
||||
size_t key1_len = strlen(key1);
|
||||
rcu_hash_add(htable, key1, key1_len, (void *)data1);
|
||||
|
||||
struct user_data *data2 = ALLOC(struct user_data, 1);
|
||||
data2->id = 102;
|
||||
char name2[64] = "127.0.0.1";
|
||||
memcpy(data2->name, name2, strlen(name2));
|
||||
char key2[64] = "http_host";
|
||||
size_t key2_len = strlen(key2);
|
||||
rcu_hash_add(htable, key2, key2_len, (void *)data2);
|
||||
|
||||
/* find in hash before commit */
|
||||
void *res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
int ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
/* add del, then commit */
|
||||
rcu_hash_del(htable, key1, key1_len);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 1);
|
||||
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
/* find in hash after commit */
|
||||
res = rcu_hash_find(htable, key1, key1_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
res = rcu_hash_find(htable, key2, key2_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
ret = rcu_hash_count(htable);
|
||||
EXPECT_EQ(ret, 1);
|
||||
|
||||
ret = rcu_hash_garbage_queue_len(htable);
|
||||
EXPECT_EQ(ret, 0);
|
||||
|
||||
rcu_hash_free(htable);
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
|
||||
2
test/rule/full/2022-11-24/HTTP_URL.000001
Normal file
2
test/rule/full/2022-11-24/HTTP_URL.000001
Normal file
@@ -0,0 +1,2 @@
|
||||
0000000001
|
||||
101 1 www.baidu.com 0 0 0 1
|
||||
1
test/rule/full/index/full_config_index.000001
Normal file
1
test/rule/full/index/full_config_index.000001
Normal file
@@ -0,0 +1 @@
|
||||
HTTP_URL 1 /home/liuwentan/project/maat-v4/test/rule/full/2022-11-24/HTTP_URL.000001
|
||||
@@ -1,9 +1,9 @@
|
||||
#each column seperated by '\t'
|
||||
[
|
||||
{
|
||||
"table_id":1,
|
||||
"table_name":"HTTP_URL",
|
||||
"table_type":"expr",
|
||||
"scan_mode":"block",
|
||||
"item_id":1,
|
||||
"group_id":2,
|
||||
"rule": {
|
||||
@@ -11,21 +11,31 @@
|
||||
"expr_type":4,
|
||||
"match_method":5,
|
||||
"is_hexbin":6,
|
||||
"case_sensitive":7,
|
||||
"is_valid":8
|
||||
"is_valid":7
|
||||
}
|
||||
},
|
||||
{
|
||||
"table_id":2,
|
||||
"table_name":"TEST_IP_PLUGIN",
|
||||
"table_name":"IP_PLUGIN_TABLE",
|
||||
"table_type":"ip_plugin",
|
||||
"item_id":1,
|
||||
"group_id":2,
|
||||
"rule": {
|
||||
"ip_type":2,
|
||||
"start_ip":3,
|
||||
"end_ip":4,
|
||||
"is_valid":5
|
||||
}
|
||||
},
|
||||
{
|
||||
"table_id":3,
|
||||
"table_name":"PLUGIN_TABLE",
|
||||
"table_type":"plugin",
|
||||
"item_id":1,
|
||||
"rule": {
|
||||
"key":2,
|
||||
"tag":3,
|
||||
"is_valid":4,
|
||||
"foreign":[6,8,10]
|
||||
}
|
||||
}
|
||||
]
|
||||
16
vendor/CMakeLists.txt
vendored
16
vendor/CMakeLists.txt
vendored
@@ -54,3 +54,19 @@ add_library(hyperscan_runtime_static STATIC IMPORTED GLOBAL)
|
||||
add_dependencies(hyperscan_runtime_static hyperscan)
|
||||
set_property(TARGET hyperscan_runtime_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhs_runtime.a)
|
||||
set_property(TARGET hyperscan_runtime_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
|
||||
|
||||
# ipmatcher-1.1
|
||||
ExternalProject_Add(ipmatcher PREFIX ipmatcher
|
||||
URL ${CMAKE_CURRENT_SOURCE_DIR}/ipmatcher-v1.1.zip
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND make
|
||||
INSTALL_COMMAND make DESTDIR=<INSTALL_DIR> install
|
||||
BUILD_IN_SOURCE 1)
|
||||
|
||||
ExternalProject_Get_Property(ipmatcher INSTALL_DIR)
|
||||
file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
|
||||
|
||||
add_library(ipmatcher-static STATIC IMPORTED GLOBAL)
|
||||
add_dependencies(ipmatcher-static ipmatcher)
|
||||
set_property(TARGET ipmatcher-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/ipmatcher.a)
|
||||
set_property(TARGET ipmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
|
||||
|
||||
BIN
vendor/ipmatcher-v1.1.zip
vendored
Normal file
BIN
vendor/ipmatcher-v1.1.zip
vendored
Normal file
Binary file not shown.
Reference in New Issue
Block a user