unfinished work
This commit is contained in:
liuwentan
@@ -8,6 +8,8 @@ set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} "-Wall")
|
||||
|
||||
include_directories(include)
|
||||
|
||||
enable_testing()
|
||||
|
||||
#add_subdirectory(source_dir [binary_dir] [EXCLUDE_FROM_ALL])
|
||||
add_subdirectory(vendor)
|
||||
add_subdirectory(src)
|
||||
|
||||
12
deps/sds/Changelog
vendored
Normal file
12
deps/sds/Changelog
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
Version 2.0
|
||||
===
|
||||
|
||||
* Better memory usage. A bit slower in certain workloads.
|
||||
* sdscatfmt() replacement for sdscatprintf() for speed critical code added.
|
||||
* Ability to easily switch allocator just changing sdsalloc.h
|
||||
* No longer binary compatible with SDS v1.0.
|
||||
|
||||
Version 1.0
|
||||
===
|
||||
|
||||
* Initial SDS stand alone verison.
|
||||
24
deps/sds/LICENSE
vendored
Normal file
24
deps/sds/LICENSE
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
Copyright (c) 2006-2014, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
8
deps/sds/Makefile
vendored
Normal file
8
deps/sds/Makefile
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
all: sds-test
|
||||
|
||||
sds-test: sds.c sds.h testhelp.h
|
||||
$(CC) -o sds-test sds.c -Wall -std=c99 -pedantic -O2 -DSDS_TEST_MAIN
|
||||
@echo ">>> Type ./sds-test to run the sds.c unit tests."
|
||||
|
||||
clean:
|
||||
rm -f sds-test
|
||||
917
deps/sds/README.md
vendored
Normal file
917
deps/sds/README.md
vendored
Normal file
@@ -0,0 +1,917 @@
|
||||
Simple Dynamic Strings
|
||||
===
|
||||
|
||||
**Notes about version 2**: this is an updated version of SDS in an attempt
|
||||
to finally unify Redis, Disque, Hiredis, and the stand alone SDS versions.
|
||||
This version is **NOT* binary compatible** with SDS verison 1, but the API
|
||||
is 99% compatible so switching to the new lib should be trivial.
|
||||
|
||||
Note that this version of SDS may be a slower with certain workloads, but
|
||||
uses less memory compared to V1 since header size is dynamic and depends to
|
||||
the string to alloc.
|
||||
|
||||
Moreover it includes a few more API functions, notably `sdscatfmt` which
|
||||
is a faster version of `sdscatprintf` that can be used for the simpler
|
||||
cases in order to avoid the libc `printf` family functions performance
|
||||
penalty.
|
||||
|
||||
How SDS strings work
|
||||
===
|
||||
|
||||
SDS is a string library for C designed to augment the limited libc string
|
||||
handling functionalities by adding heap allocated strings that are:
|
||||
|
||||
* Simpler to use.
|
||||
* Binary safe.
|
||||
* Computationally more efficient.
|
||||
* But yet... Compatible with normal C string functions.
|
||||
|
||||
This is achieved using an alternative design in which instead of using a C
|
||||
structure to represent a string, we use a binary prefix that is stored
|
||||
before the actual pointer to the string that is returned by SDS to the user.
|
||||
|
||||
+--------+-------------------------------+-----------+
|
||||
| Header | Binary safe C alike string... | Null term |
|
||||
+--------+-------------------------------+-----------+
|
||||
|
|
||||
`-> Pointer returned to the user.
|
||||
|
||||
Because of meta data stored before the actual returned pointer as a prefix,
|
||||
and because of every SDS string implicitly adding a null term at the end of
|
||||
the string regardless of the actual content of the string, SDS strings work
|
||||
well together with C strings and the user is free to use them interchangeably
|
||||
with other std C string functions that access the string in read-only.
|
||||
|
||||
SDS was a C string I developed in the past for my everyday C programming needs,
|
||||
later it was moved into Redis where it is used extensively and where it was
|
||||
modified in order to be suitable for high performance operations. Now it was
|
||||
extracted from Redis and forked as a stand alone project.
|
||||
|
||||
Because of its many years life inside Redis, SDS provides both higher level
|
||||
functions for easy strings manipulation in C, but also a set of low level
|
||||
functions that make it possible to write high performance code without paying
|
||||
a penalty for using an higher level string library.
|
||||
|
||||
Advantages and disadvantages of SDS
|
||||
===
|
||||
|
||||
Normally dynamic string libraries for C are implemented using a structure
|
||||
that defines the string. The structure has a pointer field that is managed
|
||||
by the string function, so it looks like this:
|
||||
|
||||
```c
|
||||
struct yourAverageStringLibrary {
|
||||
char *buf;
|
||||
size_t len;
|
||||
... possibly more fields here ...
|
||||
};
|
||||
```
|
||||
|
||||
SDS strings as already mentioned don't follow this schema, and are instead
|
||||
a single allocation with a prefix that lives *before* the address actually
|
||||
returned for the string.
|
||||
|
||||
There are advantages and disadvantages with this approach over the traditional
|
||||
approach:
|
||||
|
||||
**Disadvantage #1**: many functions return the new string as value, since sometimes SDS requires to create a new string with more space, so the most SDS API calls look like this:
|
||||
|
||||
```c
|
||||
s = sdscat(s,"Some more data");
|
||||
```
|
||||
|
||||
As you can see `s` is used as input for `sdscat` but is also set to the value
|
||||
returned by the SDS API call, since we are not sure if the call modified the
|
||||
SDS string we passed or allocated a new one. Not remembering to assign back
|
||||
the return value of `sdscat` or similar functions to the variable holding
|
||||
the SDS string will result in a bug.
|
||||
|
||||
**Disadvantage #2**: if an SDS string is shared in different places in your program you have to modify all the references when you modify the string. However most of the times when you need to share SDS strings it is much better to encapsulate them into structures with a `reference count` otherwise it is too easy to incur into memory leaks.
|
||||
|
||||
**Advantage #1**: you can pass SDS strings to functions designed for C functions without accessing a struct member or calling a function, like this:
|
||||
|
||||
```c
|
||||
printf("%s\n", sds_string);
|
||||
```
|
||||
|
||||
In most other libraries this will be something like:
|
||||
|
||||
```c
|
||||
printf("%s\n", string->buf);
|
||||
```
|
||||
|
||||
Or:
|
||||
|
||||
```c
|
||||
printf("%s\n", getStringPointer(string));
|
||||
```
|
||||
|
||||
**Advantage #2**: accessing individual chars is straightforward. C is a low level language so this is an important operation in many programs. With SDS strings accessing individual chars is very natural:
|
||||
|
||||
```c
|
||||
printf("%c %c\n", s[0], s[1]);
|
||||
```
|
||||
|
||||
With other libraries your best chance is to assign `string->buf` (or call the function to get the string pointer) to a `char` pointer and work with this. However since the other libraries may reallocate the buffer implicitly every time you call a function that may modify the string you have to get a reference to the buffer again.
|
||||
|
||||
**Advantage #3**: single allocation has better cache locality. Usually when you access a string created by a string library using a structure, you have two different allocations for the structure representing the string, and the actual buffer holding the string. Over the time the buffer is reallocated, and it is likely that it ends in a totally different part of memory compared to the structure itself. Since modern programs performances are often dominated by cache misses, SDS may perform better in many workloads.
|
||||
|
||||
SDS basics
|
||||
===
|
||||
|
||||
The type of SDS strings is just the char pointer `char *`. However SDS defines
|
||||
an `sds` type as alias of `char *` in its header file: you should use the
|
||||
`sds` type in order to make sure you remember that a given variable in your
|
||||
program holds an SDS string and not a C string, however this is not mandatory.
|
||||
|
||||
This is the simplest SDS program you can write that does something:
|
||||
|
||||
```c
|
||||
sds mystring = sdsnew("Hello World!");
|
||||
printf("%s\n", mystring);
|
||||
sdsfree(mystring);
|
||||
|
||||
output> Hello World!
|
||||
```
|
||||
|
||||
The above small program already shows a few important things about SDS:
|
||||
|
||||
* SDS strings are created, and heap allocated, via the `sdsnew()` function, or other similar functions that we'll see in a moment.
|
||||
* SDS strings can be passed to `printf()` like any other C string.
|
||||
* SDS strings require to be freed with `sdsfree()`, since they are heap allocated.
|
||||
|
||||
Creating SDS strings
|
||||
---
|
||||
|
||||
```c
|
||||
sds sdsnewlen(const void *init, size_t initlen);
|
||||
sds sdsnew(const char *init);
|
||||
sds sdsempty(void);
|
||||
sds sdsdup(const sds s);
|
||||
```
|
||||
|
||||
There are many ways to create SDS strings:
|
||||
|
||||
* The `sdsnew` function creates an SDS string starting from a C null terminated string. We already saw how it works in the above example.
|
||||
* The `sdsnewlen` function is similar to `sdsnew` but instead of creating the string assuming that the input string is null terminated, it gets an additional length parameter. This way you can create a string using binary data:
|
||||
|
||||
```c
|
||||
char buf[3];
|
||||
sds mystring;
|
||||
|
||||
buf[0] = 'A';
|
||||
buf[1] = 'B';
|
||||
buf[2] = 'C';
|
||||
mystring = sdsnewlen(buf,3);
|
||||
printf("%s of len %d\n", mystring, (int) sdslen(mystring));
|
||||
|
||||
output> ABC of len 3
|
||||
```
|
||||
|
||||
Note: `sdslen` return value is casted to `int` because it returns a `size_t`
|
||||
type. You can use the right `printf` specifier instead of casting.
|
||||
|
||||
* The `sdsempty()` function creates an empty zero-length string:
|
||||
|
||||
```c
|
||||
sds mystring = sdsempty();
|
||||
printf("%d\n", (int) sdslen(mystring));
|
||||
|
||||
output> 0
|
||||
```
|
||||
|
||||
* The `sdsdup()` function duplicates an already existing SDS string:
|
||||
|
||||
```c
|
||||
sds s1, s2;
|
||||
|
||||
s1 = sdsnew("Hello");
|
||||
s2 = sdsdup(s1);
|
||||
printf("%s %s\n", s1, s2);
|
||||
|
||||
output> Hello Hello
|
||||
```
|
||||
|
||||
Obtaining the string length
|
||||
---
|
||||
|
||||
```c
|
||||
size_t sdslen(const sds s);
|
||||
```
|
||||
|
||||
In the examples above we already used the `sdslen` function in order to get
|
||||
the length of the string. This function works like `strlen` of the libc
|
||||
except that:
|
||||
|
||||
* It runs in constant time since the length is stored in the prefix of SDS strings, so calling `sdslen` is not expensive even when called with very large strings.
|
||||
* The function is binary safe like any other SDS string function, so the length is the true length of the string regardless of the content, there is no problem if the string includes null term characters in the middle.
|
||||
|
||||
As an example of the binary safeness of SDS strings, we can run the following
|
||||
code:
|
||||
|
||||
```c
|
||||
sds s = sdsnewlen("A\0\0B",4);
|
||||
printf("%d\n", (int) sdslen(s));
|
||||
|
||||
output> 4
|
||||
```
|
||||
|
||||
Note that SDS strings are always null terminated at the end, so even in that
|
||||
case `s[4]` will be a null term, however printing the string with `printf`
|
||||
would result in just `"A"` to be printed since libc will treat the SDS string
|
||||
like a normal C string.
|
||||
|
||||
Destroying strings
|
||||
---
|
||||
|
||||
```c
|
||||
void sdsfree(sds s);
|
||||
```
|
||||
|
||||
The destroy an SDS string there is just to call `sdsfree` with the string
|
||||
pointer. Note that even empty strings created with `sdsempty` need to be
|
||||
destroyed as well otherwise they'll result into a memory leak.
|
||||
|
||||
The function `sdsfree` does not perform any operation if instead of an SDS
|
||||
string pointer, `NULL` is passed, so you don't need to check for `NULL` explicitly before calling it:
|
||||
|
||||
```c
|
||||
if (string) sdsfree(string); /* Not needed. */
|
||||
sdsfree(string); /* Same effect but simpler. */
|
||||
```
|
||||
|
||||
Concatenating strings
|
||||
---
|
||||
|
||||
Concatenating strings to other strings is likely the operation you will end
|
||||
using the most with a dynamic C string library. SDS provides different
|
||||
functions to concatenate strings to existing strings.
|
||||
|
||||
```c
|
||||
sds sdscatlen(sds s, const void *t, size_t len);
|
||||
sds sdscat(sds s, const char *t);
|
||||
```
|
||||
|
||||
The main string concatenation functions are `sdscatlen` and `sdscat` that are
|
||||
identical, the only difference being that `sdscat` does not have an explicit
|
||||
length argument since it expects a null terminated string.
|
||||
|
||||
```c
|
||||
sds s = sdsempty();
|
||||
s = sdscat(s, "Hello ");
|
||||
s = sdscat(s, "World!");
|
||||
printf("%s\n", s);
|
||||
|
||||
output> Hello World!
|
||||
```
|
||||
|
||||
Sometimes you want to cat an SDS string to another SDS string, so you don't
|
||||
need to specify the length, but at the same time the string does not need to
|
||||
be null terminated but can contain any binary data. For this there is a
|
||||
special function:
|
||||
|
||||
```c
|
||||
sds sdscatsds(sds s, const sds t);
|
||||
```
|
||||
|
||||
Usage is straightforward:
|
||||
|
||||
```c
|
||||
sds s1 = sdsnew("aaa");
|
||||
sds s2 = sdsnew("bbb");
|
||||
s1 = sdscatsds(s1,s2);
|
||||
sdsfree(s2);
|
||||
printf("%s\n", s1);
|
||||
|
||||
output> aaabbb
|
||||
```
|
||||
|
||||
Sometimes you don't want to append any special data to the string, but you want
|
||||
to make sure that there are at least a given number of bytes composing the
|
||||
whole string.
|
||||
|
||||
```c
|
||||
sds sdsgrowzero(sds s, size_t len);
|
||||
```
|
||||
|
||||
The `sdsgrowzero` function will do nothing if the current string length is
|
||||
already `len` bytes, otherwise it will enlarge the string to `len` just padding
|
||||
it with zero bytes.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello");
|
||||
s = sdsgrowzero(s,6);
|
||||
s[5] = '!'; /* We are sure this is safe because of sdsgrowzero() */
|
||||
printf("%s\n', s);
|
||||
|
||||
output> Hello!
|
||||
```
|
||||
|
||||
Formatting strings
|
||||
---
|
||||
|
||||
There is a special string concatenation function that accepts a `printf` alike
|
||||
format specifier and cats the formatted string to the specified string.
|
||||
|
||||
```c
|
||||
sds sdscatprintf(sds s, const char *fmt, ...) {
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```c
|
||||
sds s;
|
||||
int a = 10, b = 20;
|
||||
s = sdsnew("The sum is: ");
|
||||
s = sdscatprintf(s,"%d+%d = %d",a,b,a+b);
|
||||
```
|
||||
|
||||
Often you need to create SDS string directly from `printf` format specifiers.
|
||||
Because `sdscatprintf` is actually a function that concatenates strings, all
|
||||
you need is to concatenate your string to an empty string:
|
||||
|
||||
|
||||
```c
|
||||
char *name = "Anna";
|
||||
int loc = 2500;
|
||||
sds s;
|
||||
s = sdscatprintf(sdsempty(), "%s wrote %d lines of LISP\n", name, loc);
|
||||
```
|
||||
|
||||
You can use `sdscatprintf` in order to convert numbers into SDS strings:
|
||||
|
||||
```c
|
||||
int some_integer = 100;
|
||||
sds num = sdscatprintf(sdsempty(),"%d\n", some_integer);
|
||||
```
|
||||
|
||||
However this is slow and we have a special function to make it efficient.
|
||||
|
||||
Fast number to string operations
|
||||
---
|
||||
|
||||
Creating an SDS string from an integer may be a common operation in certain
|
||||
kind of programs, and while you may do this with `sdscatprintf` the performance
|
||||
hit is big, so SDS provides a specialized function.
|
||||
|
||||
```c
|
||||
sds sdsfromlonglong(long long value);
|
||||
```
|
||||
|
||||
Use it like this:
|
||||
|
||||
```c
|
||||
sds s = sdsfromlonglong(10000);
|
||||
printf("%d\n", (int) sdslen(s));
|
||||
|
||||
output> 5
|
||||
```
|
||||
|
||||
Trimming strings and getting ranges
|
||||
---
|
||||
|
||||
String trimming is a common operation where a set of characters are
|
||||
removed from the left and the right of the string. Another useful operation
|
||||
regarding strings is the ability to just take a range out of a larger
|
||||
string.
|
||||
|
||||
```c
|
||||
void sdstrim(sds s, const char *cset);
|
||||
void sdsrange(sds s, int start, int end);
|
||||
```
|
||||
|
||||
SDS provides both the operations with the `sdstrim` and `sdsrange` functions.
|
||||
However note that both functions work differently than most functions modifying
|
||||
SDS strings since the return value is void: basically those functions always
|
||||
destructively modify the passed SDS string, never allocating a new one, because
|
||||
both trimming and ranges will never need more room: the operations can only
|
||||
remove characters from the original string.
|
||||
|
||||
Because of this behavior, both functions are fast and don't involve reallocation.
|
||||
|
||||
This is an example of string trimming where newlines and spaces are removed
|
||||
from an SDS strings:
|
||||
|
||||
```c
|
||||
sds s = sdsnew(" my string\n\n ");
|
||||
sdstrim(s," \n");
|
||||
printf("-%s-\n",s);
|
||||
|
||||
output> -my string-
|
||||
```
|
||||
|
||||
Basically `sdstrim` takes the SDS string to trim as first argument, and a
|
||||
null terminated set of characters to remove from left and right of the string.
|
||||
The characters are removed as long as they are not interrupted by a character
|
||||
that is not in the list of characters to trim: this is why the space between
|
||||
`"my"` and `"string"` was preserved in the above example.
|
||||
|
||||
Taking ranges is similar, but instead to take a set of characters, it takes
|
||||
to indexes, representing the start and the end as specified by zero-based
|
||||
indexes inside the string, to obtain the range that will be retained.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello World!");
|
||||
sdsrange(s,1,4);
|
||||
printf("-%s-\n");
|
||||
|
||||
output> -ello-
|
||||
```
|
||||
|
||||
Indexes can be negative to specify a position starting from the end of the
|
||||
string, so that `-1` means the last character, `-2` the penultimate, and so forth:
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Hello World!");
|
||||
sdsrange(s,6,-1);
|
||||
printf("-%s-\n");
|
||||
sdsrange(s,0,-2);
|
||||
printf("-%s-\n");
|
||||
|
||||
output> -World!-
|
||||
output> -World-
|
||||
```
|
||||
|
||||
`sdsrange` is very useful when implementing networking servers processing
|
||||
a protocol or sending messages. For example the following code is used
|
||||
implementing the write handler of the Redis Cluster message bus between
|
||||
nodes:
|
||||
|
||||
```c
|
||||
void clusterWriteHandler(..., int fd, void *privdata, ...) {
|
||||
clusterLink *link = (clusterLink*) privdata;
|
||||
ssize_t nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf));
|
||||
if (nwritten <= 0) {
|
||||
/* Error handling... */
|
||||
}
|
||||
sdsrange(link->sndbuf,nwritten,-1);
|
||||
... more code here ...
|
||||
}
|
||||
```
|
||||
|
||||
Every time the socket of the node we want to send the message to is writable
|
||||
we attempt to write as much bytes as possible, and we use `sdsrange` in order
|
||||
to remove from the buffer what was already sent.
|
||||
|
||||
The function to queue new messages to send to some node in the cluster will
|
||||
simply use `sdscatlen` in order to put more data in the send buffer.
|
||||
|
||||
Note that the Redis Cluster bus implements a binary protocol, but since SDS
|
||||
is binary safe this is not a problem, so the goal of SDS is not just to provide
|
||||
an high level string API for the C programmer but also dynamically allocated
|
||||
buffers that are easy to manage.
|
||||
|
||||
String copying
|
||||
---
|
||||
|
||||
The most dangerous and infamus function of the standard C library is probably
|
||||
`strcpy`, so perhaps it is funny how in the context of better designed dynamic
|
||||
string libraries the concept of copying strings is almost irrelevant. Usually
|
||||
what you do is to create strings with the content you want, or concatenating
|
||||
more content as needed.
|
||||
|
||||
However SDS features a string copy function that is useful in performance
|
||||
critical code sections, however I guess its practical usefulness is limited
|
||||
as the function never managed to get called in the context of the 50k
|
||||
lines of code composing the Redis code base.
|
||||
|
||||
```c
|
||||
sds sdscpylen(sds s, const char *t, size_t len);
|
||||
sds sdscpy(sds s, const char *t);
|
||||
```
|
||||
|
||||
The string copy function of SDS is called `sdscpylen` and works like that:
|
||||
|
||||
```c
|
||||
s = sdsnew("Hello World!");
|
||||
s = sdscpylen(s,"Hello Superman!",15);
|
||||
```
|
||||
|
||||
As you can see the function receives as input the SDS string `s`, but also
|
||||
returns an SDS string. This is common to many SDS functions that modify the
|
||||
string: this way the returned SDS string may be the original one modified
|
||||
or a newly allocated one (for example if there was not enough room in the
|
||||
old SDS string).
|
||||
|
||||
The `sdscpylen` will simply replace what was in the old SDS string with the
|
||||
new data you pass using the pointer and length argument. There is a similar
|
||||
function called `sdscpy` that does not need a length but expects a null
|
||||
terminated string instead.
|
||||
|
||||
You may wonder why it makes sense to have a string copy function in the
|
||||
SDS library, since you can simply create a new SDS string from scratch
|
||||
with the new value instead of copying the value in an existing SDS string.
|
||||
The reason is efficiency: `sdsnewlen` will always allocate a new string
|
||||
while `sdscpylen` will try to reuse the existing string if there is enough
|
||||
room to old the new content specified by the user, and will allocate a new
|
||||
one only if needed.
|
||||
|
||||
Quoting strings
|
||||
---
|
||||
|
||||
In order to provide consistent output to the program user, or for debugging
|
||||
purposes, it is often important to turn a string that may contain binary
|
||||
data or special characters into a quoted string. Here for quoted string
|
||||
we mean the common format for String literals in programming source code.
|
||||
However today this format is also part of the well known serialization formats
|
||||
like JSON and CSV, so it definitely escaped the simple goal of representing
|
||||
literals strings in the source code of programs.
|
||||
|
||||
An example of quoted string literal is the following:
|
||||
|
||||
```c
|
||||
"\x00Hello World\n"
|
||||
```
|
||||
|
||||
The first byte is a zero byte while the last byte is a newline, so there are
|
||||
two non alphanumerical characters inside the string.
|
||||
|
||||
SDS uses a concatenation function for this goal, that concatenates to an
|
||||
existing string the quoted string representation of the input string.
|
||||
|
||||
```c
|
||||
sds sdscatrepr(sds s, const char *p, size_t len);
|
||||
```
|
||||
|
||||
The `scscatrepr` (where `repr` means *representation*) follows the usualy
|
||||
SDS string function rules accepting a char pointer and a length, so you can
|
||||
use it with SDS strings, normal C strings by using strlen() as `len` argument,
|
||||
or binary data. The following is an example usage:
|
||||
|
||||
```c
|
||||
sds s1 = sdsnew("abcd");
|
||||
sds s2 = sdsempty();
|
||||
s[1] = 1;
|
||||
s[2] = 2;
|
||||
s[3] = '\n';
|
||||
s2 = sdscatrepr(s2,s1,sdslen(s1));
|
||||
printf("%s\n", s2);
|
||||
|
||||
output> "a\x01\x02\n"
|
||||
```
|
||||
|
||||
This is the rules `sdscatrepr` uses for conversion:
|
||||
|
||||
* `\` and `"` are quoted with a backslash.
|
||||
* It quotes special characters `'\n'`, `'\r'`, `'\t'`, `'\a'` and `'\b'`.
|
||||
* All the other non printable characters not passing the `isprint` test are quoted in `\x..` form, that is: backslash followed by `x` followed by two digit hex number representing the character byte value.
|
||||
* The function always adds initial and final double quotes characters.
|
||||
|
||||
There is an SDS function that is able to perform the reverse conversion and is
|
||||
documented in the *Tokenization* section below.
|
||||
|
||||
Tokenization
|
||||
---
|
||||
|
||||
Tokenization is the process of splitting a larger string into smaller strings.
|
||||
In this specific case, the split is performed specifying another string that
|
||||
acts as separator. For example in the following string there are two substrings
|
||||
that are separated by the `|-|` separator:
|
||||
|
||||
```
|
||||
foo|-|bar|-|zap
|
||||
```
|
||||
|
||||
A more common separator that consists of a single character is the comma:
|
||||
|
||||
```
|
||||
foo,bar,zap
|
||||
```
|
||||
|
||||
In many progrems it is useful to process a line in order to obtain the sub
|
||||
strings it is composed of, so SDS provides a function that returns an
|
||||
array of SDS strings given a string and a separator.
|
||||
|
||||
```c
|
||||
sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count);
|
||||
void sdsfreesplitres(sds *tokens, int count);
|
||||
```
|
||||
|
||||
As usually the function can work with both SDS strings or normal C strings.
|
||||
The first two arguments `s` and `len` specify the string to tokenize, and the
|
||||
other two arguments `sep` and `seplen` the separator to use during the
|
||||
tokenization. The final argument `count` is a pointer to an integer that will
|
||||
be set to the number of tokens (sub strings) returned.
|
||||
|
||||
The return value is a heap allocated array of SDS strings.
|
||||
|
||||
```c
|
||||
sds *tokens;
|
||||
int count, j;
|
||||
|
||||
sds line = sdsnew("Hello World!");
|
||||
tokens = sdssplitlen(line,sdslen(line)," ",1,&count);
|
||||
|
||||
for (j = 0; j < count; j++)
|
||||
printf("%s\n", tokens[j]);
|
||||
sdsfreesplitres(tokens,count);
|
||||
|
||||
output> Hello
|
||||
output> World!
|
||||
```
|
||||
|
||||
The returned array is heap allocated, and the single elements of the array
|
||||
are normal SDS strings. You can free everything calling `sdsfreesplitres`
|
||||
as in the example. Alternativey you are free to release the array yourself
|
||||
using the `free` function and use and/or free the individual SDS strings
|
||||
as usually.
|
||||
|
||||
A valid approach is to set the array elements you reused in some way to
|
||||
`NULL`, and use `sdsfreesplitres` to free all the rest.
|
||||
|
||||
Command line oriented tokenization
|
||||
---
|
||||
|
||||
Splitting by a separator is a useful operation, but usually it is not enough
|
||||
to perform one of the most common tasks involving some non trivial string
|
||||
manipulation, that is, implementing a **Command Line Interface** for a program.
|
||||
|
||||
This is why SDS also provides an additional function that allows you to split
|
||||
arguments provided by the user via the keyboard in an interactive manner, or
|
||||
via a file, network, or any other mean, into tokens.
|
||||
|
||||
```c
|
||||
sds *sdssplitargs(const char *line, int *argc);
|
||||
```
|
||||
|
||||
The `sdssplitargs` function returns an array of SDS strings exactly like
|
||||
`sdssplitlen`. The function to free the result is also identical, and is
|
||||
`sdsfreesplitres`. The difference is in the way the tokenization is performed.
|
||||
|
||||
For example if the input is the following line:
|
||||
|
||||
```
|
||||
call "Sabrina" and "Mark Smith\n"
|
||||
```
|
||||
|
||||
The function will return the following tokens:
|
||||
|
||||
* "call"
|
||||
* "Sabrina"
|
||||
* "and"
|
||||
* "Mark Smith\n"
|
||||
|
||||
Basically different tokens need to be separated by one or more spaces, and
|
||||
every single token can also be a quoted string in the same format that
|
||||
`sdscatrepr` is able to emit.
|
||||
|
||||
String joining
|
||||
---
|
||||
|
||||
There are two functions doing the reverse of tokenization by joining strings
|
||||
into a single one.
|
||||
|
||||
```c
|
||||
sds sdsjoin(char **argv, int argc, char *sep, size_t seplen);
|
||||
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
|
||||
```
|
||||
|
||||
The two functions take as input an array of strings of length `argc` and
|
||||
a separator and its length, and produce as output an SDS string consisting
|
||||
of all the specified strings separated by the specified separator.
|
||||
|
||||
The difference between `sdsjoin` and `sdsjoinsds` is that the former accept
|
||||
C null terminated strings as input while the latter requires all the strings
|
||||
in the array to be SDS strings. However because of this only `sdsjoinsds` is
|
||||
able to deal with binary data.
|
||||
|
||||
```c
|
||||
char *tokens[3] = {"foo","bar","zap"};
|
||||
sds s = sdsjoin(tokens,3,"|",1);
|
||||
printf("%s\n", s);
|
||||
|
||||
output> foo|bar|zap
|
||||
```
|
||||
|
||||
Error handling
|
||||
---
|
||||
|
||||
All the SDS functions that return an SDS pointer may also return `NULL` on
|
||||
out of memory, this is basically the only check you need to perform.
|
||||
|
||||
However many modern C programs handle out of memory simply aborting the program
|
||||
so you may want to do this as well by wrapping `malloc` and other related
|
||||
memory allocation calls directly.
|
||||
|
||||
SDS internals and advanced usage
|
||||
===
|
||||
|
||||
At the very beginning of this documentation it was explained how SDS strings
|
||||
are allocated, however the prefix stored before the pointer returned to the
|
||||
user was classified as an *header* without further details. For an advanced
|
||||
usage it is better to dig more into the internals of SDS and show the
|
||||
structure implementing it:
|
||||
|
||||
```c
|
||||
struct sdshdr {
|
||||
int len;
|
||||
int free;
|
||||
char buf[];
|
||||
};
|
||||
```
|
||||
|
||||
As you can see, the structure may resemble the one of a conventional string
|
||||
library, however the `buf` field of the structure is different since it is
|
||||
not a pointer but an array without any length declared, so `buf` actually
|
||||
points at the first byte just after the `free` integer. So in order to create
|
||||
an SDS string we just allocate a piece of memory that is as large as the
|
||||
`sdshdr` structure plus the length of our string, plus an additional byte
|
||||
for the mandatory null term that every SDS string has.
|
||||
|
||||
The `len` field of the structure is quite obvious, and is the current length
|
||||
of the SDS string, always computed every time the string is modified via
|
||||
SDS function calls. The `free` field instead represents the amount of free
|
||||
memory in the current allocation that can be used to store more characters.
|
||||
|
||||
So the actual SDS layout is this one:
|
||||
|
||||
+------------+------------------------+-----------+---------------\
|
||||
| Len | Free | H E L L O W O R L D \n | Null term | Free space \
|
||||
+------------+------------------------+-----------+---------------\
|
||||
|
|
||||
`-> Pointer returned to the user.
|
||||
|
||||
You may wonder why there is some free space at the end of the string, it
|
||||
looks like a waste. Actually after a new SDS string is created, there is no
|
||||
free space at the end at all: the allocation will be as small as possible to
|
||||
just hold the header, string, and null term. However other access patterns
|
||||
will create extra free space at the end, like in the following program:
|
||||
|
||||
```c
|
||||
s = sdsempty();
|
||||
s = sdscat(s,"foo");
|
||||
s = sdscat(s,"bar");
|
||||
s = sdscat(s,"123");
|
||||
```
|
||||
|
||||
Since SDS tries to be efficient it can't afford to reallocate the string every
|
||||
time new data is appended, since this would be very inefficient, so it uses
|
||||
the **preallocation of some free space** every time you enlarge the string.
|
||||
|
||||
The preallocation algorithm used is the following: every time the string
|
||||
is reallocated in order to hold more bytes, the actual allocation size performed
|
||||
is two times the minimum required. So for instance if the string currently
|
||||
is holding 30 bytes, and we concatenate 2 more bytes, instead of allocating 32
|
||||
bytes in total SDS will allocate 64 bytes.
|
||||
|
||||
However there is an hard limit to the allocation it can perform ahead, and is
|
||||
defined by `SDS_MAX_PREALLOC`. SDS will never allocate more than 1MB of
|
||||
additional space (by default, you can change this default).
|
||||
|
||||
Shrinking strings
|
||||
---
|
||||
|
||||
```c
|
||||
sds sdsRemoveFreeSpace(sds s);
|
||||
size_t sdsAllocSize(sds s);
|
||||
```
|
||||
|
||||
Sometimes there are class of programs that require to use very little memory.
|
||||
After strings concatenations, trimming, ranges, the string may end having
|
||||
a non trivial amount of additional space at the end.
|
||||
|
||||
It is possible to resize a string back to its minimal size in order to hold
|
||||
the current content by using the function `sdsRemoveFreeSpace`.
|
||||
|
||||
```c
|
||||
s = sdsRemoveFreeSpace(s);
|
||||
```
|
||||
|
||||
There is also a function that can be used in order to get the size of the
|
||||
total allocation for a given string, and is called `sdsAllocSize`.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("Ladies and gentlemen");
|
||||
s = sdscat(s,"... welcome to the C language.");
|
||||
printf("%d\n", (int) sdsAllocSize(s));
|
||||
s = sdsRemoveFreeSpace(s);
|
||||
printf("%d\n", (int) sdsAllocSize(s));
|
||||
|
||||
output> 109
|
||||
output> 59
|
||||
```
|
||||
|
||||
NOTE: SDS Low level API use cammelCase in order to warn you that you are playing with the fire.
|
||||
|
||||
Manual modifications of SDS strings
|
||||
---
|
||||
|
||||
void sdsupdatelen(sds s);
|
||||
|
||||
Sometimes you may want to hack with an SDS string manually, without using
|
||||
SDS functions. In the following example we implicitly change the length
|
||||
of the string, however we want the logical length to reflect the null terminated
|
||||
C string.
|
||||
|
||||
The function `sdsupdatelen` does just that, updating the internal length
|
||||
information for the specified string to the length obtained via `strlen`.
|
||||
|
||||
```c
|
||||
sds s = sdsnew("foobar");
|
||||
s[2] = '\0';
|
||||
printf("%d\n", sdslen(s));
|
||||
sdsupdatelen(s);
|
||||
printf("%d\n", sdslen(s));
|
||||
|
||||
output> 6
|
||||
output> 2
|
||||
```
|
||||
|
||||
Sharing SDS strings
|
||||
---
|
||||
|
||||
If you are writing a program in which it is advantageous to share the same
|
||||
SDS string across different data structures, it is absolutely advised to
|
||||
encapsulate SDS strings into structures that remember the number of references
|
||||
of the string, with functions to increment and decrement the number of references.
|
||||
|
||||
This approach is a memory management technique called *reference counting* and
|
||||
in the context of SDS has two advantages:
|
||||
|
||||
* It is less likely that you'll create memory leaks or bugs due to non freeing SDS strings or freeing already freed strings.
|
||||
* You'll not need to update every reference to an SDS string when you modify it (since the new SDS string may point to a different memory location).
|
||||
|
||||
While this is definitely a very common programming technique I'll outline
|
||||
the basic ideas here. You create a structure like that:
|
||||
|
||||
```c
|
||||
struct mySharedString {
|
||||
int refcount;
|
||||
sds string;
|
||||
}
|
||||
```
|
||||
|
||||
When new strings are created, the structure is allocated and returned with
|
||||
`refcount` set to 1. The you have two functions to change the reference count
|
||||
of the shared string:
|
||||
|
||||
* `incrementStringRefCount` will simply increment `refcount` of 1 in the structure. It will be called every time you add a reference to the string on some new data structure, variable, or whatever.
|
||||
* `decrementStringRefCount` is used when you remove a reference. This function is however special since when the `refcount` drops to zero, it automatically frees the SDS string, and the `mySharedString` structure as well.
|
||||
|
||||
Interactions with heap checkers
|
||||
---
|
||||
|
||||
Because SDS returns pointers into the middle of memory chunks allocated with
|
||||
`malloc`, heap checkers may have issues, however:
|
||||
|
||||
* The popular Valgrind program will detect SDS strings are *possibly lost* memory and never as *definitely lost*, so it is easy to tell if there is a leak or not. I used Valgrind with Redis for years and every real leak was consistently detected as "definitely lost".
|
||||
* OSX instrumentation tools don't detect SDS strings as leaks but are able to correctly handle pointers pointing to the middle of memory chunks.
|
||||
|
||||
Zero copy append from syscalls
|
||||
----
|
||||
|
||||
At this point you should have all the tools to dig more inside the SDS
|
||||
library by reading the source code, however there is an interesting pattern
|
||||
you can mount using the low level API exported, that is used inside Redis
|
||||
in order to improve performances of the networking code.
|
||||
|
||||
Using `sdsIncrLen()` and `sdsMakeRoomFor()` it is possible to mount the
|
||||
following schema, to cat bytes coming from the kernel to the end of an
|
||||
sds string without copying into an intermediate buffer:
|
||||
|
||||
```c
|
||||
oldlen = sdslen(s);
|
||||
s = sdsMakeRoomFor(s, BUFFER_SIZE);
|
||||
nread = read(fd, s+oldlen, BUFFER_SIZE);
|
||||
... check for nread <= 0 and handle it ...
|
||||
sdsIncrLen(s, nread);
|
||||
```
|
||||
|
||||
`sdsIncrLen` is documented inside the source code of `sds.c`.
|
||||
|
||||
Embedding SDS into your project
|
||||
===
|
||||
|
||||
This is as simple as copying the following files inside your
|
||||
project:
|
||||
|
||||
* sds.c
|
||||
* sds.h
|
||||
* sdsalloc.h
|
||||
|
||||
The source code is small and every C99 compiler should deal with
|
||||
it without issues.
|
||||
|
||||
Using a different allocator for SDS
|
||||
===
|
||||
|
||||
Internally sds.c uses the allocator defined into `sdsalloc.h`. This header
|
||||
file just defines macros for malloc, realloc and free, and by default libc
|
||||
`malloc()`, `realloc()` and `free()` are used. Just edit this file in order
|
||||
to change the name of the allocation functions.
|
||||
|
||||
The program using SDS can call the SDS allocator in order to manipulate
|
||||
SDS pointers (usually not needed but sometimes the program may want to
|
||||
do advanced things) by using the API exported by SDS in order to call the
|
||||
allocator used. This is especially useful when the program linked to SDS
|
||||
is using a different allocator compared to what SDS is using.
|
||||
|
||||
The API to access the allocator used by SDS is composed of three functions: `sds_malloc()`, `sds_realloc()` and `sds_free()`.
|
||||
|
||||
Credits and license
|
||||
===
|
||||
|
||||
SDS was created by Salvatore Sanfilippo and is released under the BDS two clause license. See the LICENSE file in this source distribution for more information.
|
||||
|
||||
Oran Agra improved SDS version 2 by adding dynamic sized headers in order to
|
||||
save memory for small strings and allow strings greater than 4GB.
|
||||
1300
deps/sds/sds.c
vendored
Normal file
1300
deps/sds/sds.c
vendored
Normal file
File diff suppressed because it is too large
Load Diff
283
deps/sds/sds.h
vendored
Normal file
283
deps/sds/sds.h
vendored
Normal file
@@ -0,0 +1,283 @@
|
||||
/* SDSLib 2.0 -- A C dynamic strings library
|
||||
*
|
||||
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* Copyright (c) 2015, Oran Agra
|
||||
* Copyright (c) 2015, Redis Labs, Inc
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __SDS_H
|
||||
#define __SDS_H
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#define SDS_MAX_PREALLOC (1024*1024)
|
||||
extern const char *SDS_NOINIT;
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef char *sds;
|
||||
|
||||
/* Note: sdshdr5 is never used, we just access the flags byte directly.
|
||||
* However is here to document the layout of type 5 SDS strings. */
|
||||
struct __attribute__ ((__packed__)) sdshdr5 {
|
||||
unsigned char flags; /* 3 lsb of type, and 5 msb of string length */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr8 {
|
||||
uint8_t len; /* used */
|
||||
uint8_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr16 {
|
||||
uint16_t len; /* used */
|
||||
uint16_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr32 {
|
||||
uint32_t len; /* used */
|
||||
uint32_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
struct __attribute__ ((__packed__)) sdshdr64 {
|
||||
uint64_t len; /* used */
|
||||
uint64_t alloc; /* excluding the header and null terminator */
|
||||
unsigned char flags; /* 3 lsb of type, 5 unused bits */
|
||||
char buf[];
|
||||
};
|
||||
|
||||
#define SDS_TYPE_5 0
|
||||
#define SDS_TYPE_8 1
|
||||
#define SDS_TYPE_16 2
|
||||
#define SDS_TYPE_32 3
|
||||
#define SDS_TYPE_64 4
|
||||
#define SDS_TYPE_MASK 7
|
||||
#define SDS_TYPE_BITS 3
|
||||
#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)));
|
||||
#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))))
|
||||
#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS)
|
||||
|
||||
static inline size_t sdslen(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
return SDS_TYPE_5_LEN(flags);
|
||||
case SDS_TYPE_8:
|
||||
return SDS_HDR(8,s)->len;
|
||||
case SDS_TYPE_16:
|
||||
return SDS_HDR(16,s)->len;
|
||||
case SDS_TYPE_32:
|
||||
return SDS_HDR(32,s)->len;
|
||||
case SDS_TYPE_64:
|
||||
return SDS_HDR(64,s)->len;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline size_t sdsavail(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5: {
|
||||
return 0;
|
||||
}
|
||||
case SDS_TYPE_8: {
|
||||
SDS_HDR_VAR(8,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_16: {
|
||||
SDS_HDR_VAR(16,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_32: {
|
||||
SDS_HDR_VAR(32,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
case SDS_TYPE_64: {
|
||||
SDS_HDR_VAR(64,s);
|
||||
return sh->alloc - sh->len;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void sdssetlen(sds s, size_t newlen) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
{
|
||||
unsigned char *fp = ((unsigned char*)s)-1;
|
||||
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
|
||||
}
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->len = newlen;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->len = newlen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void sdsinclen(sds s, size_t inc) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
{
|
||||
unsigned char *fp = ((unsigned char*)s)-1;
|
||||
unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc;
|
||||
*fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS);
|
||||
}
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->len += inc;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->len += inc;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* sdsalloc() = sdsavail() + sdslen() */
|
||||
static inline size_t sdsalloc(const sds s) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
return SDS_TYPE_5_LEN(flags);
|
||||
case SDS_TYPE_8:
|
||||
return SDS_HDR(8,s)->alloc;
|
||||
case SDS_TYPE_16:
|
||||
return SDS_HDR(16,s)->alloc;
|
||||
case SDS_TYPE_32:
|
||||
return SDS_HDR(32,s)->alloc;
|
||||
case SDS_TYPE_64:
|
||||
return SDS_HDR(64,s)->alloc;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void sdssetalloc(sds s, size_t newlen) {
|
||||
unsigned char flags = s[-1];
|
||||
switch(flags&SDS_TYPE_MASK) {
|
||||
case SDS_TYPE_5:
|
||||
/* Nothing to do, this type has no total allocation info. */
|
||||
break;
|
||||
case SDS_TYPE_8:
|
||||
SDS_HDR(8,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_16:
|
||||
SDS_HDR(16,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_32:
|
||||
SDS_HDR(32,s)->alloc = newlen;
|
||||
break;
|
||||
case SDS_TYPE_64:
|
||||
SDS_HDR(64,s)->alloc = newlen;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sds sdsnewlen(const void *init, size_t initlen);
|
||||
sds sdsnew(const char *init);
|
||||
sds sdsempty(void);
|
||||
sds sdsdup(const sds s);
|
||||
void sdsfree(sds s);
|
||||
sds sdsgrowzero(sds s, size_t len);
|
||||
sds sdscatlen(sds s, const void *t, size_t len);
|
||||
sds sdscat(sds s, const char *t);
|
||||
sds sdscatsds(sds s, const sds t);
|
||||
sds sdscpylen(sds s, const char *t, size_t len);
|
||||
sds sdscpy(sds s, const char *t);
|
||||
|
||||
sds sdscatvprintf(sds s, const char *fmt, va_list ap);
|
||||
#ifdef __GNUC__
|
||||
sds sdscatprintf(sds s, const char *fmt, ...)
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
#else
|
||||
sds sdscatprintf(sds s, const char *fmt, ...);
|
||||
#endif
|
||||
|
||||
sds sdscatfmt(sds s, char const *fmt, ...);
|
||||
sds sdstrim(sds s, const char *cset);
|
||||
void sdsrange(sds s, ssize_t start, ssize_t end);
|
||||
void sdsupdatelen(sds s);
|
||||
void sdsclear(sds s);
|
||||
int sdscmp(const sds s1, const sds s2);
|
||||
sds *sdssplitlen(const char *s, ssize_t len, const char *sep, int seplen, int *count);
|
||||
void sdsfreesplitres(sds *tokens, int count);
|
||||
void sdstolower(sds s);
|
||||
void sdstoupper(sds s);
|
||||
sds sdsfromlonglong(long long value);
|
||||
sds sdscatrepr(sds s, const char *p, size_t len);
|
||||
sds *sdssplitargs(const char *line, int *argc);
|
||||
sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen);
|
||||
sds sdsjoin(char **argv, int argc, char *sep);
|
||||
sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen);
|
||||
|
||||
/* Low level functions exposed to the user API */
|
||||
sds sdsMakeRoomFor(sds s, size_t addlen);
|
||||
void sdsIncrLen(sds s, ssize_t incr);
|
||||
sds sdsRemoveFreeSpace(sds s);
|
||||
size_t sdsAllocSize(sds s);
|
||||
void *sdsAllocPtr(sds s);
|
||||
|
||||
/* Export the allocator used by SDS to the program using SDS.
|
||||
* Sometimes the program SDS is linked to, may use a different set of
|
||||
* allocators, but may want to allocate or free things that SDS will
|
||||
* respectively free or allocate. */
|
||||
void *sds_malloc(size_t size);
|
||||
void *sds_realloc(void *ptr, size_t size);
|
||||
void sds_free(void *ptr);
|
||||
|
||||
#ifdef REDIS_TEST
|
||||
int sdsTest(int argc, char *argv[]);
|
||||
#endif
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
42
deps/sds/sdsalloc.h
vendored
Normal file
42
deps/sds/sdsalloc.h
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
/* SDSLib 2.0 -- A C dynamic strings library
|
||||
*
|
||||
* Copyright (c) 2006-2015, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* Copyright (c) 2015, Oran Agra
|
||||
* Copyright (c) 2015, Redis Labs, Inc
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/* SDS allocator selection.
|
||||
*
|
||||
* This file is used in order to change the SDS allocator at compile time.
|
||||
* Just define the following defines to what you want to use. Also add
|
||||
* the include of your alternate allocator if needed (not needed in order
|
||||
* to use the default libc allocator). */
|
||||
|
||||
#define s_malloc malloc
|
||||
#define s_realloc realloc
|
||||
#define s_free free
|
||||
57
deps/sds/testhelp.h
vendored
Normal file
57
deps/sds/testhelp.h
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
/* This is a really minimal testing framework for C.
|
||||
*
|
||||
* Example:
|
||||
*
|
||||
* test_cond("Check if 1 == 1", 1==1)
|
||||
* test_cond("Check if 5 > 10", 5 > 10)
|
||||
* test_report()
|
||||
*
|
||||
* ----------------------------------------------------------------------------
|
||||
*
|
||||
* Copyright (c) 2010-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
* * Neither the name of Redis nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without
|
||||
* specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
||||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
* POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __TESTHELP_H
|
||||
#define __TESTHELP_H
|
||||
|
||||
int __failed_tests = 0;
|
||||
int __test_num = 0;
|
||||
#define test_cond(descr,_c) do { \
|
||||
__test_num++; printf("%d - %s: ", __test_num, descr); \
|
||||
if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \
|
||||
} while(0);
|
||||
#define test_report() do { \
|
||||
printf("%d tests, %d passed, %d failed\n", __test_num, \
|
||||
__test_num-__failed_tests, __failed_tests); \
|
||||
if (__failed_tests) { \
|
||||
printf("=== WARNING === We have failed tests here...\n"); \
|
||||
exit(1); \
|
||||
} \
|
||||
} while(0);
|
||||
|
||||
#endif
|
||||
@@ -1,40 +1,75 @@
|
||||
#pragma once
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* Maat: Deep Packet Inspection Policy Framework
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
* Maat is the Goddess of truth and justice in ancient Egyptian concept.
|
||||
* Her feather was the measure that determined whether the souls (considered
|
||||
* to reside in the heart) of the departed would reach the paradise of afterlife
|
||||
* successfully.
|
||||
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_H_
|
||||
#define _MAAT_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <netinet/in.h>
|
||||
|
||||
struct maat_feather;
|
||||
/* maat instance handle */
|
||||
struct maat;
|
||||
|
||||
/* network-order */
|
||||
struct ipv4_tuple4{
|
||||
uint32_t saddr; /* network order */
|
||||
uint32_t daddr; /* network order */
|
||||
uint16_t source; /* network order */
|
||||
uint16_t dest; /* network order */
|
||||
struct maat_rule {
|
||||
int rule_id;
|
||||
};
|
||||
|
||||
#define IPV6_ADDR_LEN (sizeof(struct in6_addr))
|
||||
|
||||
struct ipv6_tuple4
|
||||
{
|
||||
uint8_t saddr[IPV6_ADDR_LEN] ;
|
||||
uint8_t daddr[IPV6_ADDR_LEN] ;
|
||||
uint16_t source; /* network order */
|
||||
uint16_t dest; /* network order */
|
||||
enum ip_type {
|
||||
IP_TYPE_V4,
|
||||
IP_TYPE_V6
|
||||
};
|
||||
|
||||
struct maat_scan_state;
|
||||
int maat_scan_ipv4(struct maat_feather *feather, int table_id, struct maat_scan_state *state,
|
||||
const struct ipv4_tuple4 *tuple4, unsigned int *matched_ids, size_t n_match_id);
|
||||
struct ip_data {
|
||||
enum ip_type type;
|
||||
union {
|
||||
uint32_t ipv4;
|
||||
uint32_t ipv6[4];
|
||||
};
|
||||
};
|
||||
|
||||
struct maat_options;
|
||||
struct maat_options* maat_options_new(void);
|
||||
int maat_options_set_worker_thread_number(struct maat_options *opts, size_t nr_worker_threads);
|
||||
|
||||
struct maat *maat_new(struct maat_options options, const char* table_info_path);
|
||||
void maat_free(struct maat *instance);
|
||||
|
||||
int maat_table_get_id(struct maat *instance, const char *table_name);
|
||||
|
||||
struct maat_state;
|
||||
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
|
||||
unsigned int intval, int results[], size_t *n_result,
|
||||
struct maat_state *state);
|
||||
|
||||
int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
|
||||
const struct ip_data *ip, int results[], size_t *n_result,
|
||||
struct maat_state *state);
|
||||
|
||||
int maat_scan_string(struct maat *instance, int table_id, int thread_id,
|
||||
const char *data, size_t data_len, int results[], size_t *n_result,
|
||||
struct maat_state *state);
|
||||
|
||||
struct maat_stream;
|
||||
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id);
|
||||
|
||||
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len,
|
||||
int results[], size_t *n_result, struct maat_state *state);
|
||||
|
||||
void maat_scan_stream_close(struct maat_stream **stream);
|
||||
|
||||
void maat_state_reset(struct maat_state *state);
|
||||
|
||||
int maat_scan_string(struct maat_feather *feather, int table_id, struct maat_scan_state *state,
|
||||
const char *data, size_t length);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
@@ -8,9 +8,11 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
|
||||
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
|
||||
|
||||
add_definitions(-fPIC)
|
||||
set(MAAT_SRC maat_api.cpp)
|
||||
set(MAAT_SRC maat_api.cpp bool_matcher.cpp adapter_hs.cpp rcu_hash.cpp maat_garbage_collection.cpp)
|
||||
|
||||
include_directories(${PROJECT_SOURCE_DIR}/include/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/deps/)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
|
||||
|
||||
# Static Library Output
|
||||
add_library(maat_frame_static STATIC ${MAAT_SRC})
|
||||
|
||||
534
src/adapter_hs.cpp
Normal file
534
src/adapter_hs.cpp
Normal file
@@ -0,0 +1,534 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: adapter_hs.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stddef.h>
|
||||
#include <hs/hs.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "uthash/utarray.h"
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_utils.h"
|
||||
#include "bool_matcher.h"
|
||||
|
||||
struct adpt_hs_compile_data {
|
||||
unsigned int *ids;
|
||||
unsigned int *flags;
|
||||
char **patterns;
|
||||
size_t *pattern_lens;
|
||||
unsigned int n_patterns;
|
||||
};
|
||||
|
||||
/* adapter_hs runtime */
|
||||
struct adapter_hs_runtime {
|
||||
hs_database_t *literal_db;
|
||||
hs_database_t *regex_db;
|
||||
|
||||
hs_scratch_t **scratchs;
|
||||
size_t scratch_size;
|
||||
|
||||
struct bool_matcher *bm;
|
||||
};
|
||||
|
||||
/* adapter_hs instance */
|
||||
struct adapter_hs {
|
||||
size_t nr_worker_threads;
|
||||
size_t n_expr;
|
||||
size_t n_patterns;
|
||||
struct adapter_hs_runtime *hs_rt;
|
||||
};
|
||||
|
||||
struct adapter_hs_stream {
|
||||
int thread_id;
|
||||
size_t n_expr;
|
||||
size_t n_patterns;
|
||||
hs_stream_t *literal_stream;
|
||||
hs_stream_t *regex_stream;
|
||||
struct adapter_hs_runtime *hs_rt;
|
||||
UT_array *pattern_id_set;
|
||||
};
|
||||
|
||||
int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type)
|
||||
{
|
||||
hs_database_t *database = NULL;
|
||||
hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads);
|
||||
|
||||
if (max_pattern_type == PATTERN_TYPE_STR) {
|
||||
database = hs_rt->literal_db;
|
||||
} else {
|
||||
database = hs_rt->regex_db;
|
||||
}
|
||||
|
||||
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
|
||||
// log_error("ERROR: Unable to allocate scratch space. Exiting.\n");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (size_t i = 1; i < nr_worker_threads; i++) {
|
||||
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error("Unable to clone scratch prototype");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
|
||||
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error("Unable to query scratch size");
|
||||
hs_free_database(database);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief build hs block database for literal string and regex expression respectively
|
||||
*
|
||||
* @retval 0(success) -1(failed)
|
||||
*/
|
||||
int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
|
||||
struct adpt_hs_compile_data *literal_cd,
|
||||
struct adpt_hs_compile_data *regex_cd,
|
||||
int scan_mode)
|
||||
{
|
||||
hs_error_t err;
|
||||
hs_compile_error_t *compile_err = NULL;
|
||||
|
||||
if (NULL == hs_rt) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (literal_cd != NULL) {
|
||||
err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags,
|
||||
literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns,
|
||||
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
if (compile_err) {
|
||||
printf("compile error: %s", compile_err->message);
|
||||
}
|
||||
|
||||
hs_free_compile_error(compile_err);
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
|
||||
if (regex_cd != NULL) {
|
||||
err = hs_compile_ext_multi((const char *const *)regex_cd->patterns, regex_cd->flags,
|
||||
regex_cd->ids, NULL, regex_cd->n_patterns,
|
||||
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
if (compile_err) {
|
||||
printf("compile error: %s", compile_err->message);
|
||||
}
|
||||
hs_free_compile_error(compile_err);
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
error:
|
||||
if (hs_rt->literal_db != NULL) {
|
||||
hs_free_database(hs_rt->literal_db);
|
||||
hs_rt->literal_db = NULL;
|
||||
}
|
||||
|
||||
if (hs_rt->regex_db != NULL) {
|
||||
hs_free_database(hs_rt->regex_db);
|
||||
hs_rt->regex_db = NULL;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns)
|
||||
{
|
||||
struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1);
|
||||
hs_cd->patterns = ALLOC(char *, n_patterns);
|
||||
hs_cd->pattern_lens = ALLOC(size_t, n_patterns);
|
||||
hs_cd->ids = ALLOC(unsigned int, n_patterns);
|
||||
hs_cd->flags = ALLOC(unsigned int, n_patterns);
|
||||
|
||||
return hs_cd;
|
||||
}
|
||||
|
||||
void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patterns)
|
||||
{
|
||||
if (NULL == hs_cd) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (hs_cd->patterns != NULL) {
|
||||
for (size_t i = 0; i < n_patterns; i++) {
|
||||
free(hs_cd->patterns[i]);
|
||||
}
|
||||
|
||||
free(hs_cd->patterns);
|
||||
free(hs_cd->pattern_lens);
|
||||
free(hs_cd->ids);
|
||||
free(hs_cd->flags);
|
||||
}
|
||||
|
||||
free(hs_cd);
|
||||
}
|
||||
|
||||
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array)
|
||||
{
|
||||
if ((scan_mode != SCAN_MODE_BLOCK && scan_mode != SCAN_MODE_STREAM) ||
|
||||
0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
|
||||
/* log_error("input parameters illegal!"); */
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* get the sum of pattern */
|
||||
size_t literal_pattern_num = 0;
|
||||
size_t regex_pattern_num = 0;
|
||||
|
||||
for (size_t i = 0; i < n_expr_array; i++) {
|
||||
if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
|
||||
//log_error("the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
|
||||
if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) {
|
||||
literal_pattern_num++;
|
||||
} else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) {
|
||||
regex_pattern_num++;
|
||||
} else {
|
||||
/* log_error("unknown pattern type: %d\n", expr_array[i].patterns[j].type); */
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct adpt_hs_compile_data *literal_cd = NULL;
|
||||
struct adpt_hs_compile_data *regex_cd = NULL;
|
||||
if (literal_pattern_num > 0) {
|
||||
literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
|
||||
}
|
||||
|
||||
if (regex_pattern_num > 0) {
|
||||
regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
|
||||
}
|
||||
|
||||
uint32_t literal_index = 0;
|
||||
uint32_t regex_index = 0;
|
||||
uint32_t pattern_id = 0;
|
||||
|
||||
/* alloc exprs for bool matcher*/
|
||||
struct bool_expr *exprs = ALLOC(struct bool_expr, n_expr_array);
|
||||
|
||||
/* populate adpt_hs_compile_data and bool_expr */
|
||||
for (size_t i = 0; i < n_expr_array; i++) {
|
||||
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
|
||||
size_t pat_len = 0;
|
||||
|
||||
if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) {
|
||||
literal_cd->ids[literal_index] = pattern_id;
|
||||
literal_cd->flags[literal_index] = HS_FLAG_CASELESS;
|
||||
|
||||
pat_len = expr_array[i].patterns[j].pat_len;
|
||||
literal_cd->pattern_lens[literal_index] = pat_len;
|
||||
literal_cd->patterns[literal_index] = ALLOC(char, pat_len);
|
||||
memcpy(literal_cd->patterns[literal_index],
|
||||
expr_array[i].patterns[j].pat,
|
||||
expr_array[i].patterns[j].pat_len);
|
||||
literal_index++;
|
||||
} else {
|
||||
regex_cd->ids[regex_index] = pattern_id;
|
||||
regex_cd->flags[regex_index] = HS_FLAG_CASELESS;
|
||||
|
||||
pat_len = expr_array[i].patterns[j].pat_len;
|
||||
regex_cd->pattern_lens[regex_index] = pat_len;
|
||||
regex_cd->patterns[regex_index] = ALLOC(char, pat_len);
|
||||
memcpy(regex_cd->patterns[regex_index],
|
||||
expr_array[i].patterns[j].pat,
|
||||
expr_array[i].patterns[j].pat_len);
|
||||
regex_index++;
|
||||
}
|
||||
exprs[i].items[j].item_id = pattern_id;
|
||||
pattern_id++;
|
||||
}
|
||||
exprs[i].expr_id = expr_array[i].expr_id;
|
||||
exprs[i].item_num = expr_array[i].n_patterns;
|
||||
}
|
||||
|
||||
if (literal_cd != NULL) {
|
||||
literal_cd->n_patterns = literal_index;
|
||||
}
|
||||
|
||||
if (regex_cd != NULL) {
|
||||
regex_cd->n_patterns = regex_index;
|
||||
}
|
||||
|
||||
int ret = -1;
|
||||
int max_patterns_type = 0;
|
||||
size_t mem_size = 0;
|
||||
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
|
||||
|
||||
hs_instance->nr_worker_threads = nr_worker_threads;
|
||||
hs_instance->n_patterns = pattern_id;
|
||||
hs_instance->n_expr = n_expr_array;
|
||||
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
|
||||
|
||||
/* create bool matcher */
|
||||
hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size);
|
||||
if (hs_instance->hs_rt->bm != NULL) {
|
||||
/* log_info("Adapter_hs module: build bool matcher of %u expressions with %u bytes memory.",
|
||||
n_expr_array, mem_size); */
|
||||
} else {
|
||||
/* log_error("Adapter_hs module: build bool matcher failed."); */
|
||||
goto error;
|
||||
}
|
||||
free(exprs);
|
||||
|
||||
/* build hs database */
|
||||
ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode);
|
||||
if (ret < 0) {
|
||||
//log_error()
|
||||
goto error;
|
||||
}
|
||||
|
||||
if (literal_cd != NULL) {
|
||||
adpt_hs_compile_data_free(literal_cd, literal_index);
|
||||
}
|
||||
|
||||
if (regex_cd != NULL) {
|
||||
adpt_hs_compile_data_free(regex_cd, regex_index);
|
||||
}
|
||||
|
||||
/* which pattern type has more patterns, use it as hs_alloc_scratch's input parameter */
|
||||
if (literal_pattern_num > regex_pattern_num) {
|
||||
max_patterns_type = PATTERN_TYPE_STR;
|
||||
} else {
|
||||
max_patterns_type = PATTERN_TYPE_REG;
|
||||
}
|
||||
|
||||
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type);
|
||||
if (ret < 0) {
|
||||
// log_error
|
||||
goto error;
|
||||
}
|
||||
|
||||
return hs_instance;
|
||||
error:
|
||||
adapter_hs_destroy(hs_instance);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void adapter_hs_destroy(struct adapter_hs *hs_instance)
|
||||
{
|
||||
if (NULL == hs_instance) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (hs_instance->hs_rt != NULL) {
|
||||
if (hs_instance->hs_rt->literal_db != NULL) {
|
||||
hs_free_database(hs_instance->hs_rt->literal_db);
|
||||
}
|
||||
|
||||
if (hs_instance->hs_rt->regex_db != NULL) {
|
||||
hs_free_database(hs_instance->hs_rt->regex_db);
|
||||
}
|
||||
|
||||
if (hs_instance->hs_rt->scratchs != NULL) {
|
||||
for (size_t i = 0; i < hs_instance->nr_worker_threads; i++) {
|
||||
if (hs_instance->hs_rt->scratchs[i] != NULL) {
|
||||
hs_free_scratch(hs_instance->hs_rt->scratchs[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
free(hs_instance->hs_rt->scratchs);
|
||||
|
||||
if (hs_instance->hs_rt->bm != NULL) {
|
||||
bool_matcher_free(hs_instance->hs_rt->bm);
|
||||
}
|
||||
|
||||
free(hs_instance->hs_rt);
|
||||
}
|
||||
|
||||
free(hs_instance);
|
||||
}
|
||||
|
||||
static inline int compare_pattern_id(const void* a, const void* b)
|
||||
{
|
||||
long long ret= *(unsigned long long *)a - *(unsigned long long *)b;
|
||||
|
||||
if (0 == ret) {
|
||||
return 0;
|
||||
} else if (ret < 0) {
|
||||
return -1;
|
||||
} else {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
|
||||
|
||||
/**
|
||||
* @param id: pattern id
|
||||
*/
|
||||
int matched_event_cb(unsigned int id, unsigned long long from,
|
||||
unsigned long long to, unsigned int flags, void *ctx) {
|
||||
// put id in set
|
||||
UT_array *pattern_id_set = (UT_array *)ctx;
|
||||
unsigned long long pattern_id = (unsigned long long)id;
|
||||
if (utarray_find(pattern_id_set, &pattern_id, compare_pattern_id)) {
|
||||
return -1;
|
||||
}
|
||||
utarray_push_back(pattern_id_set, &pattern_id);
|
||||
utarray_sort(pattern_id_set, compare_pattern_id);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len,
|
||||
int results[], size_t *n_results)
|
||||
{
|
||||
struct adapter_hs_runtime *hs_rt = hs_instance->hs_rt;
|
||||
hs_scratch_t *scratch = hs_rt->scratchs[thread_id];
|
||||
UT_array *pattern_id_set;
|
||||
hs_error_t err;
|
||||
|
||||
utarray_new(pattern_id_set, &ut_pattern_id_icd);
|
||||
utarray_reserve(pattern_id_set, hs_instance->n_patterns);
|
||||
|
||||
if (hs_rt->literal_db != NULL) {
|
||||
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
|
||||
if (err != HS_SUCCESS) {
|
||||
//log_error()
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (hs_rt->regex_db != NULL) {
|
||||
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
|
||||
if (err != HS_SUCCESS) {
|
||||
//log_error()
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t pattern_set_size = utarray_len(pattern_id_set);
|
||||
unsigned long long items[pattern_set_size];
|
||||
memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
|
||||
for (size_t i = 0; i < pattern_set_size; i++) {
|
||||
items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i);
|
||||
}
|
||||
|
||||
size_t matched_index = 0;
|
||||
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
|
||||
size_t bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_instance->n_expr);
|
||||
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
|
||||
results[matched_index] = bool_matcher_results[matched_index].expr_id;
|
||||
}
|
||||
*n_results = bool_matcher_ret;
|
||||
|
||||
free(bool_matcher_results);
|
||||
utarray_free(pattern_id_set);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
|
||||
{
|
||||
struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1);
|
||||
hs_error_t err;
|
||||
|
||||
hs_stream->thread_id = thread_id;
|
||||
hs_stream->n_expr = hs_instance->n_expr;
|
||||
hs_stream->n_patterns = hs_instance->n_patterns;
|
||||
hs_stream->hs_rt = hs_instance->hs_rt;
|
||||
utarray_new(hs_stream->pattern_id_set, &ut_pattern_id_icd);
|
||||
utarray_reserve(hs_stream->pattern_id_set, hs_stream->n_patterns);
|
||||
|
||||
if (hs_instance->hs_rt->literal_db != NULL) {
|
||||
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (hs_instance->hs_rt->regex_db != NULL) {
|
||||
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream);
|
||||
if (err != HS_SUCCESS) {
|
||||
// log_error
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return hs_stream;
|
||||
}
|
||||
|
||||
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
|
||||
int results[], size_t *n_results)
|
||||
{
|
||||
hs_error_t err;
|
||||
|
||||
int thread_id = hs_stream->thread_id;
|
||||
if (hs_stream->literal_stream != NULL) {
|
||||
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
|
||||
matched_event_cb, hs_stream->pattern_id_set);
|
||||
if (err != HS_SUCCESS) {
|
||||
//log_error()
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (hs_stream->regex_stream != NULL) {
|
||||
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id],
|
||||
matched_event_cb, hs_stream->pattern_id_set);
|
||||
if (err != HS_SUCCESS) {
|
||||
//log_error()
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
size_t pattern_set_size = utarray_len(hs_stream->pattern_id_set);
|
||||
unsigned long long items[pattern_set_size];
|
||||
memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
|
||||
for (size_t i = 0; i < pattern_set_size; i++) {
|
||||
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i);
|
||||
}
|
||||
|
||||
size_t matched_index = 0;
|
||||
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
|
||||
size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr);
|
||||
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
|
||||
results[matched_index] = bool_matcher_results[matched_index].expr_id;
|
||||
}
|
||||
*n_results = bool_matcher_ret;
|
||||
|
||||
free(bool_matcher_results);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
|
||||
{
|
||||
int thread_id = hs_stream->thread_id;
|
||||
|
||||
hs_close_stream(hs_stream->literal_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
|
||||
hs_close_stream(hs_stream->regex_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL);
|
||||
utarray_free(hs_stream->pattern_id_set);
|
||||
|
||||
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
|
||||
hs_stream->hs_rt = NULL;
|
||||
free(hs_stream);
|
||||
}
|
||||
216
src/bool_matcher.cpp
Normal file
216
src/bool_matcher.cpp
Normal file
@@ -0,0 +1,216 @@
|
||||
#include "bool_matcher.h"
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
using namespace std;
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
struct bool_expr_item
|
||||
{
|
||||
size_t item_num;
|
||||
struct bool_item * items;
|
||||
};
|
||||
|
||||
struct bool_matcher
|
||||
{
|
||||
unsigned int bool_expr_num;
|
||||
struct bool_expr_match * bool_expr_ids;
|
||||
struct bool_expr_item * bool_expr_items;
|
||||
unsigned int bool_item_num;
|
||||
unsigned long long * bool_items;
|
||||
unsigned int * mapped_ptr;
|
||||
unsigned int * mapped_ids;
|
||||
unsigned int bitmap_size;
|
||||
unsigned char * bitmap;
|
||||
};
|
||||
|
||||
bool operator<(const struct bool_item & lhs, const struct bool_item & rhs)
|
||||
{
|
||||
return lhs.item_id<rhs.item_id;
|
||||
}
|
||||
|
||||
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size)
|
||||
{
|
||||
if(exprs==NULL || expr_num==0) return NULL;
|
||||
|
||||
unsigned int mem_bytes=0;
|
||||
|
||||
struct bool_matcher * matcher=new struct bool_matcher;
|
||||
mem_bytes+=sizeof(bool_matcher);
|
||||
|
||||
matcher->bool_expr_num=(unsigned int)expr_num;
|
||||
matcher->bool_expr_ids =new struct bool_expr_match[expr_num];
|
||||
matcher->bool_expr_items=new struct bool_expr_item[expr_num];
|
||||
mem_bytes+=(unsigned int)expr_num*(sizeof(struct bool_expr_match)+sizeof(struct bool_expr_item));
|
||||
for(unsigned int i=0; i<expr_num; i++)
|
||||
{
|
||||
matcher->bool_expr_ids[i].expr_id =exprs[i].expr_id;
|
||||
matcher->bool_expr_ids[i].user_tag =exprs[i].user_tag;
|
||||
matcher->bool_expr_items[i].item_num=exprs[i].item_num;
|
||||
matcher->bool_expr_items[i].items=new struct bool_item[exprs[i].item_num];
|
||||
mem_bytes+=(unsigned int)exprs[i].item_num*sizeof(struct bool_item);
|
||||
copy(exprs[i].items, exprs[i].items+exprs[i].item_num, matcher->bool_expr_items[i].items);
|
||||
sort(matcher->bool_expr_items[i].items, matcher->bool_expr_items[i].items+exprs[i].item_num);
|
||||
}
|
||||
|
||||
map<unsigned long long, unsigned int> M1;
|
||||
for(unsigned int i=0; i<expr_num; i++)
|
||||
{
|
||||
for(unsigned int j=0; j<exprs[i].item_num; j++)
|
||||
{
|
||||
if(exprs[i].items[j].not_flag==0) M1[exprs[i].items[j].item_id]++;
|
||||
}
|
||||
}
|
||||
|
||||
map< unsigned long long, vector<unsigned int> > M2;
|
||||
for(unsigned int i=0; i<expr_num; i++)
|
||||
{
|
||||
unsigned int min_count=-1;
|
||||
unsigned long long item_id;
|
||||
for(unsigned int j=0; j<exprs[i].item_num; j++)
|
||||
{
|
||||
if(exprs[i].items[j].not_flag==0)
|
||||
{
|
||||
unsigned int c=M1[exprs[i].items[j].item_id];
|
||||
if(c<min_count)
|
||||
{
|
||||
min_count=c;
|
||||
item_id=exprs[i].items[j].item_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
M2[item_id].push_back(i);
|
||||
}
|
||||
|
||||
matcher->bool_item_num=(unsigned int)M2.size();
|
||||
matcher->bool_items =new unsigned long long[M2.size()];
|
||||
matcher->mapped_ptr =new unsigned int[M2.size()+1];
|
||||
matcher->mapped_ids =new unsigned int[matcher->bool_expr_num];
|
||||
mem_bytes+=((unsigned int)M2.size()+1+matcher->bool_expr_num)*sizeof(unsigned int)+(unsigned int)M2.size()*sizeof(unsigned long long);
|
||||
|
||||
matcher->mapped_ptr[0]=0;
|
||||
map< unsigned long long, vector<unsigned int> >::const_iterator it=M2.begin();
|
||||
for(unsigned int k=0; k<M2.size(); ++k, ++it)
|
||||
{
|
||||
matcher->bool_items[k]=it->first;
|
||||
copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]);
|
||||
matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size();
|
||||
}
|
||||
|
||||
M1.clear();
|
||||
M2.clear();
|
||||
|
||||
matcher->bitmap_size=(1U<<27);
|
||||
matcher->bitmap=new unsigned char[(matcher->bitmap_size)>>3];
|
||||
mem_bytes+=(matcher->bitmap_size)>>3;
|
||||
memset(matcher->bitmap, 0, (matcher->bitmap_size)>>3);
|
||||
|
||||
for(unsigned int i=0; i<matcher->bool_item_num; i++)
|
||||
{
|
||||
unsigned int j=matcher->bool_items[i]&(matcher->bitmap_size-1);
|
||||
matcher->bitmap[j>>3]|=(1U<<(j&7));
|
||||
}
|
||||
|
||||
if(mem_size!=NULL) *mem_size=mem_bytes;
|
||||
return matcher;
|
||||
}
|
||||
|
||||
int res_comp(const void * lhs, const void * rhs)
|
||||
{
|
||||
bool_expr_match * _lhs=(bool_expr_match *)lhs;
|
||||
bool_expr_match * _rhs=(bool_expr_match *)rhs;
|
||||
return (_lhs->expr_id<_rhs->expr_id) ? 1 : -1;
|
||||
}
|
||||
|
||||
int do_match(struct bool_expr_item * expr, unsigned long long * item_ids, size_t item_num)
|
||||
{
|
||||
unsigned int i=0;
|
||||
for(unsigned int j=0; j<expr->item_num; ++j)
|
||||
{
|
||||
if(expr->items[j].not_flag==0)
|
||||
{
|
||||
while(i<item_num && item_ids[i]<expr->items[j].item_id) ++i;
|
||||
if(i==item_num || item_ids[i]>expr->items[j].item_id) return 0;
|
||||
++i;
|
||||
}
|
||||
else
|
||||
{
|
||||
while(i<item_num && item_ids[i]<expr->items[j].item_id) ++i;
|
||||
if(i<item_num && item_ids[i]==expr->items[j].item_id) return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result)
|
||||
{
|
||||
if(matcher==NULL) return -1;
|
||||
if(item_num==0) return 0;
|
||||
|
||||
// sort(item_ids, item_ids+item_num);
|
||||
// size_t J=0;
|
||||
// for(unsigned int i=1; i<item_num; i++)
|
||||
// {
|
||||
// if(item_ids[i]!=item_ids[J]) item_ids[++J]=item_ids[i];
|
||||
// }
|
||||
// item_num=J+1;
|
||||
|
||||
unsigned int r=0;
|
||||
|
||||
for(unsigned int i=0; i<item_num; i++)
|
||||
{
|
||||
unsigned int t=item_ids[i]&(matcher->bitmap_size-1);
|
||||
if((matcher->bitmap[t>>3]&(1U<<(t&7)))==0) continue;
|
||||
|
||||
int l=0, h=(int)matcher->bool_item_num-1;
|
||||
while(l<=h)
|
||||
{
|
||||
int m=(l+h)/2;
|
||||
if(item_ids[i]==matcher->bool_items[m])
|
||||
{
|
||||
for(unsigned int j=matcher->mapped_ptr[m]; j<matcher->mapped_ptr[m+1]; j++)
|
||||
{
|
||||
unsigned int idx=matcher->mapped_ids[j];
|
||||
int ret=do_match(matcher->bool_expr_items+idx, item_ids, item_num);
|
||||
if(ret==1)
|
||||
{
|
||||
if(r==n_result) goto END;
|
||||
results[r++]=matcher->bool_expr_ids[idx];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
else if(item_ids[i]<matcher->bool_items[m])
|
||||
{
|
||||
h=m-1;
|
||||
}
|
||||
else
|
||||
{
|
||||
l=m+1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
END:
|
||||
qsort(results, r, sizeof(bool_expr_match), res_comp);
|
||||
return r;
|
||||
}
|
||||
|
||||
void bool_matcher_free(struct bool_matcher * matcher)
|
||||
{
|
||||
if(matcher==NULL) return;
|
||||
|
||||
delete [] matcher->bool_expr_ids;
|
||||
for(unsigned int i=0; i<matcher->bool_expr_num; i++) delete [] matcher->bool_expr_items[i].items;
|
||||
delete [] matcher->bool_expr_items;
|
||||
|
||||
delete [] matcher->bool_items;
|
||||
delete [] matcher->mapped_ptr;
|
||||
delete [] matcher->mapped_ids;
|
||||
delete [] matcher->bitmap;
|
||||
delete matcher;
|
||||
return;
|
||||
}
|
||||
2932
src/cJSON.c
Normal file
2932
src/cJSON.c
Normal file
File diff suppressed because it is too large
Load Diff
103
src/inc_internal/adapter_hs.h
Normal file
103
src/inc_internal/adapter_hs.h
Normal file
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: adapter_hs.h
|
||||
* Description: wrapper for raw hyperscan
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _ADAPTER_HS_H_
|
||||
#define _ADAPTER_HS_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define MAX_EXPR_PATTERN_NUM 8
|
||||
|
||||
struct adapter_hs;
|
||||
|
||||
/* scan mode */
|
||||
enum {
|
||||
SCAN_MODE_BLOCK = 1,
|
||||
SCAN_MODE_STREAM,
|
||||
};
|
||||
|
||||
/* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */
|
||||
enum {
|
||||
PATTERN_TYPE_STR = 1,
|
||||
PATTERN_TYPE_REG,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
/* pattern type */
|
||||
int type;
|
||||
|
||||
/* start pointer of pattern */
|
||||
char *pat;
|
||||
/* pattern length */
|
||||
size_t pat_len;
|
||||
} scan_pattern_t;
|
||||
|
||||
/* logic AND expression, such as (pattern1 & pattern2) */
|
||||
typedef struct {
|
||||
uint32_t expr_id;
|
||||
size_t n_patterns;
|
||||
scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM];
|
||||
} and_expr_t;
|
||||
|
||||
/**
|
||||
* @brief initialize adapter_hs instance
|
||||
*
|
||||
* @param scan_mode: the following scan as block or stream mode
|
||||
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
|
||||
* @param expr_array: logic AND expression's array
|
||||
* @param n_expr_arrays: the number of logic AND expression's array
|
||||
*
|
||||
* @retval the pointer to adapter_hs instance
|
||||
*/
|
||||
struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array);
|
||||
|
||||
/**
|
||||
* @brief scan input data to match logic AND expression, return all matched expr_id
|
||||
*
|
||||
* @param instance: adapter_hs instance obtained by adapter_hs_initialize()
|
||||
* @param thread_id: the thread_id of caller
|
||||
* @param data: data to be scanned
|
||||
* @param data_len: the length of data to be scanned
|
||||
* @param results: the array of expr_id
|
||||
* @param n_results: number of elements in array of expr_id
|
||||
*/
|
||||
int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len,
|
||||
int results[], size_t *n_results);
|
||||
|
||||
/**
|
||||
* @brief destroy adapter_hs instance
|
||||
*
|
||||
* @param instance: adapter_hs instance obtained by adapter_hs_initialize()
|
||||
*/
|
||||
void adapter_hs_destroy(struct adapter_hs *instance);
|
||||
|
||||
struct adapter_hs_stream;
|
||||
/**
|
||||
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
|
||||
*
|
||||
*/
|
||||
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
|
||||
|
||||
int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
|
||||
int results[], size_t *n_results);
|
||||
|
||||
void adapter_hs_stream_close(struct adapter_hs_stream *stream);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
66
src/inc_internal/bool_matcher.h
Normal file
66
src/inc_internal/bool_matcher.h
Normal file
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
*
|
||||
* Copyright (c) 2018
|
||||
* String Algorithms Research Group
|
||||
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
|
||||
* National Engineering Laboratory for Information Security Technologies (NELIST)
|
||||
* All rights reserved
|
||||
*
|
||||
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
|
||||
* Last modification: 2021-06-12
|
||||
*
|
||||
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
|
||||
* Usage for direct or indirect commercial advantage is not allowed without
|
||||
* written permission from the authors.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef INCLUDE_BOOL_MATCHER_H
|
||||
#define INCLUDE_BOOL_MATCHER_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#define MAX_ITEMS_PER_BOOL_EXPR 8
|
||||
|
||||
/* not_flag=0表示布尔项item_id必须出现；not_flag=1表示布尔项item_id不能出现 */
|
||||
struct bool_item
|
||||
{
|
||||
unsigned long long item_id;
|
||||
unsigned char not_flag;
|
||||
};
|
||||
|
||||
/* At least one item's not_flag should be 0. */
|
||||
struct bool_expr
|
||||
{
|
||||
unsigned long long expr_id;
|
||||
void *user_tag;
|
||||
size_t item_num;
|
||||
struct bool_item items[MAX_ITEMS_PER_BOOL_EXPR];
|
||||
};
|
||||
|
||||
struct bool_expr_match
|
||||
{
|
||||
unsigned long long expr_id;
|
||||
void *user_tag;
|
||||
};
|
||||
|
||||
struct bool_matcher;
|
||||
|
||||
struct bool_matcher *bool_matcher_new(struct bool_expr *exprs, size_t expr_num, size_t *mem_size);
|
||||
|
||||
/* Returned results are sorted by expr_id in descending order. */
|
||||
// Input item_ids MUST be ASCENDING order and NO duplication.
|
||||
int bool_matcher_match(struct bool_matcher *matcher, unsigned long long *item_ids, size_t item_num, struct bool_expr_match *results, size_t n_result);
|
||||
|
||||
void bool_matcher_free(struct bool_matcher *matcher);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
277
src/inc_internal/cJSON.h
Normal file
277
src/inc_internal/cJSON.h
Normal file
@@ -0,0 +1,277 @@
|
||||
/*
|
||||
Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#ifndef cJSON__h
|
||||
#define cJSON__h
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
/* project version */
|
||||
#define CJSON_VERSION_MAJOR 1
|
||||
#define CJSON_VERSION_MINOR 7
|
||||
#define CJSON_VERSION_PATCH 7
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* cJSON Types: */
|
||||
#define cJSON_Invalid (0)
|
||||
#define cJSON_False (1 << 0)
|
||||
#define cJSON_True (1 << 1)
|
||||
#define cJSON_NULL (1 << 2)
|
||||
#define cJSON_Number (1 << 3)
|
||||
#define cJSON_String (1 << 4)
|
||||
#define cJSON_Array (1 << 5)
|
||||
#define cJSON_Object (1 << 6)
|
||||
#define cJSON_Raw (1 << 7) /* raw json */
|
||||
|
||||
#define cJSON_IsReference 256
|
||||
#define cJSON_StringIsConst 512
|
||||
|
||||
/* The cJSON structure: */
|
||||
typedef struct cJSON
|
||||
{
|
||||
/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
|
||||
struct cJSON *next;
|
||||
struct cJSON *prev;
|
||||
/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
|
||||
struct cJSON *child;
|
||||
|
||||
/* The type of the item, as above. */
|
||||
int type;
|
||||
|
||||
/* The item's string, if type==cJSON_String and type == cJSON_Raw */
|
||||
char *valuestring;
|
||||
/* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
|
||||
int valueint;
|
||||
/* The item's number, if type==cJSON_Number */
|
||||
double valuedouble;
|
||||
|
||||
/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
|
||||
char *string;
|
||||
} cJSON;
|
||||
|
||||
typedef struct cJSON_Hooks
|
||||
{
|
||||
void *(*malloc_fn)(size_t sz);
|
||||
void (*free_fn)(void *ptr);
|
||||
} cJSON_Hooks;
|
||||
|
||||
typedef int cJSON_bool;
|
||||
|
||||
#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
|
||||
#define __WINDOWS__
|
||||
#endif
|
||||
#ifdef __WINDOWS__
|
||||
|
||||
/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options:
|
||||
|
||||
CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
|
||||
CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
|
||||
CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
|
||||
|
||||
For *nix builds that support visibility attribute, you can define similar behavior by
|
||||
|
||||
setting default visibility to hidden by adding
|
||||
-fvisibility=hidden (for gcc)
|
||||
or
|
||||
-xldscope=hidden (for sun cc)
|
||||
to CFLAGS
|
||||
|
||||
then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
|
||||
|
||||
*/
|
||||
|
||||
/* export symbols by default, this is necessary for copy pasting the C and header file */
|
||||
#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
|
||||
#define CJSON_EXPORT_SYMBOLS
|
||||
#endif
|
||||
|
||||
#if defined(CJSON_HIDE_SYMBOLS)
|
||||
#define CJSON_PUBLIC(type) type __stdcall
|
||||
#elif defined(CJSON_EXPORT_SYMBOLS)
|
||||
#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall
|
||||
#elif defined(CJSON_IMPORT_SYMBOLS)
|
||||
#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall
|
||||
#endif
|
||||
#else /* !WIN32 */
|
||||
#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
|
||||
#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type
|
||||
#else
|
||||
#define CJSON_PUBLIC(type) type
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
|
||||
* This is to prevent stack overflows. */
|
||||
#ifndef CJSON_NESTING_LIMIT
|
||||
#define CJSON_NESTING_LIMIT 1000
|
||||
#endif
|
||||
|
||||
/* returns the version of cJSON as a string */
|
||||
CJSON_PUBLIC(const char*) cJSON_Version(void);
|
||||
|
||||
/* Supply malloc, realloc and free functions to cJSON */
|
||||
CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
|
||||
|
||||
/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
|
||||
/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
|
||||
/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
|
||||
/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
|
||||
|
||||
/* Render a cJSON entity to text for transfer/storage. */
|
||||
CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
|
||||
/* Render a cJSON entity to text for transfer/storage without any formatting. */
|
||||
CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
|
||||
/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
|
||||
CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
|
||||
/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
|
||||
/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
|
||||
/* Delete a cJSON entity and all subentities. */
|
||||
CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
|
||||
|
||||
/* Returns the number of items in an array (or object). */
|
||||
CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
|
||||
/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
|
||||
/* Get item "string" from object. Case insensitive. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
|
||||
/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
|
||||
CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
|
||||
|
||||
/* Check if the item is a string and return its valuestring */
|
||||
CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item);
|
||||
|
||||
/* These functions check the type of an item */
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
|
||||
|
||||
/* These calls create a cJSON item of the appropriate type. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
|
||||
/* raw json */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
|
||||
|
||||
/* Create a string where valuestring references a string so
|
||||
* it will not be freed by cJSON_Delete */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string);
|
||||
/* Create an object/arrray that only references it's elements so
|
||||
* they will not be freed by cJSON_Delete */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child);
|
||||
|
||||
/* These utilities create an Array of count items. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
|
||||
|
||||
/* Append item to the specified array/object. */
|
||||
CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
|
||||
CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
|
||||
/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
|
||||
* WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
|
||||
* writing to `item->string` */
|
||||
CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
|
||||
/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
|
||||
CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
|
||||
CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
|
||||
|
||||
/* Remove/Detatch items from Arrays/Objects. */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
|
||||
CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
|
||||
CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
|
||||
CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
|
||||
CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
|
||||
|
||||
/* Update array items. */
|
||||
CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
|
||||
CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
|
||||
CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
|
||||
CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
|
||||
|
||||
/* Duplicate a cJSON item */
|
||||
CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
|
||||
/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
|
||||
need to be released. With recurse!=0, it will duplicate any children connected to the item.
|
||||
The item->next and ->prev pointers are always zero on return from Duplicate. */
|
||||
/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
|
||||
* case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
|
||||
CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
|
||||
|
||||
|
||||
CJSON_PUBLIC(void) cJSON_Minify(char *json);
|
||||
|
||||
/* Helper functions for creating and adding items to an object at the same time.
|
||||
* They return the added item or NULL on failure. */
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name);
|
||||
CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name);
|
||||
|
||||
/* When assigning an integer value, it needs to be propagated to valuedouble too. */
|
||||
#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
|
||||
/* helper for the cJSON_SetNumberValue macro */
|
||||
CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
|
||||
#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
|
||||
|
||||
/* Macro for iterating over an array or object */
|
||||
#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
|
||||
|
||||
/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
|
||||
CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
|
||||
CJSON_PUBLIC(void) cJSON_free(void *object);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
29
src/inc_internal/maat_common.h
Normal file
29
src/inc_internal/maat_common.h
Normal file
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_common.h
|
||||
* Description: maat common entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_COMMON_H_
|
||||
#define _MAAT_COMMON_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
struct maat_options {
|
||||
size_t nr_worker_threads;
|
||||
};
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
36
src/inc_internal/maat_config_monitor.h
Normal file
36
src/inc_internal/maat_config_monitor.h
Normal file
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_config_monitor.h
|
||||
* Description: maat config monitor api
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_CONFIG_MONITOR_H_
|
||||
#define _MAAT_CONFIG_MONITOR_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define CONFIG_UPDATE_TYPE_NONE 0
|
||||
#define CONFIG_UPDATE_TYPE_FULL 1
|
||||
#define CONFIG_UPDATE_TYPE_INC 2
|
||||
|
||||
|
||||
void config_monitor_traverse(uint64_t version, const char *idx_dir,
|
||||
void (*pre_fn)(uint64_t, int, void *),
|
||||
int (*update_fn)(const char *, const char *, void *),
|
||||
void (*post_fn)(void *),
|
||||
void *u_param);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
29
src/inc_internal/maat_ex_data.h
Normal file
29
src/inc_internal/maat_ex_data.h
Normal file
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_ex_data.h
|
||||
* Description: ex data
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_EX_DATA_H_
|
||||
#define _MAAT_EX_DATA_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
struct ex_data_runtime;
|
||||
|
||||
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data));
|
||||
|
||||
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
39
src/inc_internal/maat_garbage_collection.h
Normal file
39
src/inc_internal/maat_garbage_collection.h
Normal file
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_garbage_collection.h
|
||||
* Description: maat gc
|
||||
* Authors: Zhengchao <zhengchao@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_GARBAGE_COLLECTION_H_
|
||||
#define _MAAT_GARBAGE_COLLECTION_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
struct maat_garbage_bin;
|
||||
|
||||
struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout);
|
||||
|
||||
void maat_garbage_bin_free(struct maat_garbage_bin* bin);
|
||||
|
||||
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *));
|
||||
|
||||
void maat_garbage_collect_routine(struct maat_garbage_bin* bin);
|
||||
|
||||
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin);
|
||||
|
||||
void maat_garbage_collect_by_force(struct maat_garbage_bin* bin);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
33
src/inc_internal/maat_kv.h
Normal file
33
src/inc_internal/maat_kv.h
Normal file
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_kv_map.h
|
||||
* Description: str2int map api
|
||||
* Authors: Zheng chao <zhengchao@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_KV_MAP_H_
|
||||
#define _MAAT_KV_MAP_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
struct maat_kv_store;
|
||||
|
||||
struct maat_kv_store* maat_kv_store_new(void);
|
||||
|
||||
void maat_kv_store_free(struct maat_kv_store* store);
|
||||
|
||||
int maat_kv_register(struct maat_kv_store* store, const char* key, int value);
|
||||
|
||||
int maat_kv_read(struct maat_kv_store* store, const char* key, int* value);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
74
src/inc_internal/maat_rule.h
Normal file
74
src/inc_internal/maat_rule.h
Normal file
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_rule.h
|
||||
* Description: maat rule entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_RULE_H_
|
||||
#define _MAAT_RULE_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <limits.h>
|
||||
#include <sys/time.h>
|
||||
#include <pthread.h>
|
||||
|
||||
struct maat_runtime {
|
||||
/* maat_runtime can be created and destroy dynamic, so need version info */
|
||||
uint64_t version;
|
||||
|
||||
time_t last_update_time;
|
||||
|
||||
struct maat_table_runtime_manager *table_rt_mgr;
|
||||
size_t max_table_num;
|
||||
|
||||
int max_thread_num;
|
||||
uint32_t rule_num;
|
||||
};
|
||||
|
||||
enum rule_import_type {
|
||||
RULE_IMPORT_TYPE_IRIS = 1,
|
||||
RULE_IMPORT_TYPE_MAX
|
||||
};
|
||||
|
||||
struct rule_import_iris_ctx {
|
||||
char inc_dir[NAME_MAX];
|
||||
char full_dir[NAME_MAX];
|
||||
};
|
||||
|
||||
struct maat {
|
||||
struct maat_runtime *maat_rt;
|
||||
struct maat_runtime *rebuilding_maat_rt; //TODO: creating
|
||||
//struct maat_garbage_collector *gc;
|
||||
struct maat_table_manager *table_mgr;
|
||||
|
||||
enum rule_import_type rule_import_type;
|
||||
union {
|
||||
struct rule_import_iris_ctx iris_ctx;
|
||||
};
|
||||
|
||||
int is_running;
|
||||
pthread_mutex_t background_update_mutex;
|
||||
int nr_worker_thread;
|
||||
|
||||
uint64_t maat_version;
|
||||
uint64_t last_full_version;
|
||||
pthread_t cfg_mon_thread;
|
||||
};
|
||||
|
||||
void *rule_monitor_loop(void *arg);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
40
src/inc_internal/maat_table_runtime.h
Normal file
40
src/inc_internal/maat_table_runtime.h
Normal file
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_table_runtime.h
|
||||
* Description: maat table runtime entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_TABLE_RUNTIME_H_
|
||||
#define _MAAT_TABLE_RUNTIME_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_garbage_collection.h"
|
||||
|
||||
struct maat_table_item;
|
||||
struct maat_table_runtime;
|
||||
struct maat_table_runtime_manager;
|
||||
|
||||
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* bin);
|
||||
|
||||
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr);
|
||||
|
||||
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id);
|
||||
|
||||
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt);
|
||||
|
||||
void maat_table_runtime_item_add(struct maat_table_runtime *table_rt, struct maat_table_item *table_item);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
47
src/inc_internal/maat_table_schema.h
Normal file
47
src/inc_internal/maat_table_schema.h
Normal file
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_table_schema.h
|
||||
* Description: maat table schema entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_TABLE_SCHEMA_H_
|
||||
#define _MAAT_TABLE_SCHEMA_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
|
||||
enum maat_table_type {
|
||||
TABLE_TYPE_EXPR = 0,
|
||||
TABLE_TYPE_IP_PLUGIN,
|
||||
TABLE_TYPE_MAX
|
||||
};
|
||||
|
||||
struct maat_table_schema;
|
||||
struct maat_table_manager;
|
||||
struct maat_table_item;
|
||||
|
||||
struct maat_table_manager *maat_table_manager_create(sds table_info_path);
|
||||
void maat_table_manager_destroy(struct maat_table_manager *table_mgr);
|
||||
|
||||
int maat_table_manager_get_table_id(struct maat_table_manager* table_mgr, sds table_name);
|
||||
enum maat_table_type maat_table_manager_get_table_type(struct maat_table_manager *table_mgr, int id);
|
||||
|
||||
size_t maat_table_manager_get_size(struct maat_table_manager* table_mgr);
|
||||
|
||||
struct maat_table_item *maat_table_line_to_item(sds line, struct maat_table_schema *table_schema);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
32
src/inc_internal/maat_utils.h
Normal file
32
src/inc_internal/maat_utils.h
Normal file
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_utils.h
|
||||
* Description: maat utils entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _MAAT_UTILS_H_
|
||||
#define _MAAT_UTILS_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "sds/sds.h"
|
||||
|
||||
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
|
||||
|
||||
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
59
src/inc_internal/rcu_hash.h
Normal file
59
src/inc_internal/rcu_hash.h
Normal file
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_rhash.h
|
||||
* Description: maat rcu hashtable
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef _RCU_HASH_H_
|
||||
#define _RCU_HASH_H_
|
||||
|
||||
#ifdef __cpluscplus
|
||||
extern "C"
|
||||
{
|
||||
#endif
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
|
||||
/* rcu hash table */
|
||||
struct rcu_hash_table;
|
||||
|
||||
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data));
|
||||
|
||||
void rcu_hash_free(struct rcu_hash_table *htable);
|
||||
|
||||
/**
|
||||
* @brief the data added just in updating stage
|
||||
* after call rcu_hash_commit, it in effective stage
|
||||
*/
|
||||
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data);
|
||||
|
||||
void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len);
|
||||
|
||||
/**
|
||||
* @brief find in effective nodes
|
||||
*
|
||||
* @param htable: the rcu_hash_table
|
||||
* @param key: the key used for searching in the hash table
|
||||
* @param key_len: the key's length
|
||||
*
|
||||
* @retval NULL or
|
||||
*/
|
||||
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len);
|
||||
|
||||
size_t rcu_hash_counts(struct rcu_hash_table *htable);
|
||||
|
||||
|
||||
/**
|
||||
* @brief make add/del effective
|
||||
*/
|
||||
void rcu_hash_commit(struct rcu_hash_table *htable);
|
||||
|
||||
#ifdef __cpluscplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
118
src/maat_api.cpp
118
src/maat_api.cpp
@@ -1,31 +1,121 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_api.cpp
|
||||
* Description: maat api entry
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <hs/hs.h>
|
||||
|
||||
#include "../include/maat/maat.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_rule.h"
|
||||
#include "maat_common.h"
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "sds/sds.h"
|
||||
|
||||
int maat_scan_ipv4(struct maat_feather *feather, struct maat_scan_state *state,
|
||||
const struct ipv4_tuple4 *tuple4, unsigned int *matched_ids, size_t n_match_id)
|
||||
struct maat_options* maat_options_new(void)
|
||||
{
|
||||
struct maat_options *options = ALLOC(struct maat_options, 1);
|
||||
|
||||
options->nr_worker_threads = 1;
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
int maat_options_set_worker_thread_number(struct maat_options *opts, size_t n_worker_threads)
|
||||
{
|
||||
opts->nr_worker_threads = n_worker_threads;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct maat *maat_new(struct maat_options opts, const char *table_info_path)
|
||||
{
|
||||
#if 0
|
||||
if (NULL == table_info_path) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct maat *maat_instance = ALLOC(struct maat, 1);
|
||||
|
||||
maat_instance->table_mgr = maat_table_manager_create(table_info_path);
|
||||
maat_instance->is_running = 0;
|
||||
maat_instance->maat_version = 0;
|
||||
maat_instance->last_full_version = 0;
|
||||
maat_instance->nr_worker_thread = opts.nr_worker_threads;
|
||||
|
||||
pthread_create(&(maat_instance->cfg_mon_thread), NULL, rule_monitor_loop, (void*)maat_instance);
|
||||
|
||||
return maat_instance;
|
||||
#endif
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void maat_free(struct maat *maat_instance)
|
||||
{
|
||||
|
||||
void* ret = NULL;
|
||||
pthread_join(maat_instance->cfg_mon_thread, &ret);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
int maat_table_get_id(struct maat *instance, const char *table_name)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_scan_string(struct maat_feather *feather, int table_id, struct maat_scan_state *state,
|
||||
const char *data, size_t length)
|
||||
int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
|
||||
unsigned int intval, int results[], size_t n_result,
|
||||
struct maat_state *state)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_hyperscan_compile(char *pattern)
|
||||
int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
|
||||
const struct ip_data *ip, int results[], size_t n_result,
|
||||
struct maat_state *state)
|
||||
{
|
||||
hs_database_t *database;
|
||||
hs_compile_error_t *compile_err;
|
||||
if (hs_compile(pattern, HS_FLAG_DOTALL, HS_MODE_BLOCK, NULL, &database,
|
||||
&compile_err) != HS_SUCCESS) {
|
||||
fprintf(stderr, "ERROR: Unable to compile pattern \"%s\": %s\n",
|
||||
pattern, compile_err->message);
|
||||
hs_free_compile_error(compile_err);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
|
||||
const char *data, size_t data_len, int results[], size_t *n_results,
|
||||
struct maat_state *state)
|
||||
{
|
||||
#if 0
|
||||
if (NULL == maat_instance || NULL == data ) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct maat_runtime *maat_rt = maat_instance->maat_rt;
|
||||
|
||||
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, table_id);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len,
|
||||
int results[], size_t n_result, struct maat_state *state)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
void maat_scan_stream_close(struct maat_stream **stream)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
void maat_state_reset(struct maat_state *state)
|
||||
{
|
||||
|
||||
}
|
||||
217
src/maat_config_monitor.cpp
Normal file
217
src/maat_config_monitor.cpp
Normal file
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_config_monitor.h
|
||||
* Description: maat config monitor api
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "maat_config_monitor.h"
|
||||
#include "maat_utils.h"
|
||||
|
||||
#define CM_MAX_TABLE_NUM 256
|
||||
#define MAX_CONFIG_LINE (1024 * 16)
|
||||
|
||||
struct cm_table_info_t
|
||||
{
|
||||
char table_name[NAME_MAX];
|
||||
char cfg_path[NAME_MAX];
|
||||
int cfg_num;
|
||||
char encryp_algorithm[NAME_MAX];
|
||||
};
|
||||
|
||||
/*
|
||||
@brief check if rule file updated
|
||||
@retval 0 -> CONFIG_UPDATE_TYPE_NONE
|
||||
1 -> CONFIG_UPDATE_TYPE_FULL
|
||||
2 -> CONFIG_UPDATE_TYPE_INC
|
||||
*/
|
||||
int validate_update_happened(uint64_t current_version, const char *idx_dir, char **idx_paths[], size_t *n_idx_paths)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
int cm_read_cfg_index_file(const char* path, struct cm_table_info_t* idx, int size)
|
||||
{
|
||||
int ret = 0;
|
||||
int i = 0;
|
||||
char line[MAX_CONFIG_LINE];
|
||||
struct stat file_info;
|
||||
|
||||
FILE* fp = fopen(path,"r");
|
||||
while (!feof(fp)) {
|
||||
memset(line, 0, sizeof(line));
|
||||
fgets(line, sizeof(line), fp);
|
||||
ret=sscanf(line,"%s\t%d\t%s\t%s",idx[i].table_name
|
||||
,&(idx[i].cfg_num)
|
||||
,idx[i].cfg_path
|
||||
,idx[i].encryp_algorithm);
|
||||
//jump over empty line
|
||||
if (!(ret==3||ret==4)||idx[i].cfg_num==0){
|
||||
continue;
|
||||
}
|
||||
|
||||
ret = stat(idx[i].cfg_path, &file_info);
|
||||
if (ret != 0) {
|
||||
//log_error
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
i++;
|
||||
if (i == size) {
|
||||
//log_error
|
||||
break;
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
|
||||
return i;
|
||||
}
|
||||
|
||||
const char* path2filename(const char*path)
|
||||
{
|
||||
int i=0;
|
||||
for(i=strlen(path);i>0;i--)
|
||||
{
|
||||
if(path[i]=='/')
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
return path+i+1;
|
||||
}
|
||||
|
||||
char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size)
|
||||
{
|
||||
int this_offset=0;
|
||||
const char* p;
|
||||
|
||||
//search for CRLF, aka CR '\r'(old Mac), LF '\n'(UNIX) or CRLF"\r\n" (Windows)
|
||||
p = (const char*)memchr(buff+*offset, '\n', buff_size-*offset);
|
||||
if (p == NULL) { // NOT "\n" or "\r\n"
|
||||
p = (const char*)memchr(buff+*offset, '\r', buff_size-*offset);
|
||||
}
|
||||
|
||||
if (p != NULL) { //point to next character
|
||||
p++;
|
||||
} else { //Treat rest buff has no CRLF as a line.
|
||||
p = buff + buff_size;
|
||||
}
|
||||
|
||||
this_offset = p - (buff + *offset);
|
||||
memcpy(line, buff + *offset, MIN(this_offset, line_size-1));
|
||||
|
||||
*offset += this_offset;
|
||||
line[MIN(this_offset, line_size - 1)] = '\0';
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
int cm_read_table_file(struct cm_table_info_t* index,
|
||||
int (*update_fn)(const char *, const char *, void *),
|
||||
void* u_param)
|
||||
{
|
||||
int cfg_num = 0,i =0;
|
||||
int ret = 0;
|
||||
char error_string[NAME_MAX];
|
||||
char line[MAX_CONFIG_LINE]={0};
|
||||
char *ret_str=NULL;
|
||||
char *table_file_buff=NULL;
|
||||
size_t file_sz = 0;
|
||||
size_t file_offset = 0;
|
||||
|
||||
ret = load_file_to_memory(index->cfg_path, (unsigned char **)&table_file_buff, &file_sz);
|
||||
if (ret < 0) {
|
||||
// log_error
|
||||
return -1;
|
||||
}
|
||||
|
||||
read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
|
||||
sscanf(line, "%d\n", &cfg_num);
|
||||
|
||||
if(cfg_num != index->cfg_num) {
|
||||
//log_error
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (i = 0; i < cfg_num; i++) {
|
||||
line[sizeof(line)-1]='\0';
|
||||
|
||||
ret_str=read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
|
||||
|
||||
if(ret_str==NULL) {
|
||||
//log_error
|
||||
break;
|
||||
}
|
||||
|
||||
if(line[sizeof(line)-1]!='\0') {
|
||||
//log_error
|
||||
continue;
|
||||
}
|
||||
|
||||
ret = update_fn(index->table_name, line, u_param);
|
||||
if (ret<0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
free(table_file_buff);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void config_monitor_traverse(uint64_t current_version, const char *idx_dir,
|
||||
void (*start_fn)(uint64_t, int, void *),
|
||||
int (*update_fn)(const char *, const char *, void *),
|
||||
void (*finish_fn)(void *),
|
||||
void *u_param)
|
||||
{
|
||||
size_t i = 0;
|
||||
uint64_t new_version = 0;
|
||||
char **idx_path_array = NULL;
|
||||
size_t idx_path_num = 0;
|
||||
struct cm_table_info_t table_array[CM_MAX_TABLE_NUM];
|
||||
|
||||
memset(table_array, 0, sizeof(table_array));
|
||||
|
||||
int update_type = validate_update_happened(current_version, idx_dir, &idx_path_array, &idx_path_num);
|
||||
if (update_type != CONFIG_UPDATE_TYPE_NONE) {
|
||||
for (i = 0; i < idx_path_num; i++) {
|
||||
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
|
||||
if (table_num < 0) {
|
||||
//log_error
|
||||
break;
|
||||
}
|
||||
|
||||
char str_not_care[256] = {0};
|
||||
const char* table_filename = path2filename(idx_path_array[i]);
|
||||
sscanf(table_filename,"%[a-zA-Z]_config_index.%lld", str_not_care, &new_version);
|
||||
if (start_fn != NULL) {
|
||||
start_fn(new_version, update_type, u_param);
|
||||
}
|
||||
|
||||
for (int j = 0; j < table_num; j++) {
|
||||
cm_read_table_file(table_array + j, update_fn, u_param);
|
||||
}
|
||||
|
||||
if (finish_fn != NULL) {
|
||||
finish_fn(u_param);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < idx_path_num; i++) {
|
||||
free(idx_path_array[i]);
|
||||
}
|
||||
|
||||
free(idx_path_array);
|
||||
}
|
||||
70
src/maat_ex_data.cpp
Normal file
70
src/maat_ex_data.cpp
Normal file
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_ex_data.cpp
|
||||
* Description: ex data
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
#include "uthash/utarray.h"
|
||||
#include "maat_rhash.h"
|
||||
#include "maat_utils.h"
|
||||
|
||||
typedef void ex_data_schema_new_func_t(int table_id, const char *key, const char *table_line, void* ad, long argl, void *argp);
|
||||
typedef void ex_data_schema_free_func_t(int table_id, void* ad, long argl, void *argp);
|
||||
typedef void ex_data_schema_dup_func_t(int table_id, void *to, void *from, long argl, void *argp);
|
||||
/*
|
||||
struct ex_data_schema
|
||||
{
|
||||
ex_data_schema_new_func_t* new_func;
|
||||
ex_data_schema_free_func_t* free_func;
|
||||
ex_data_schema_dup_func_t* dup_func;
|
||||
//Maat_plugin_EX_key2index_func_t* key2index_func;
|
||||
long argl;
|
||||
void *argp;
|
||||
}; */
|
||||
|
||||
struct ex_data_runtime {
|
||||
UT_array *cache_rows;
|
||||
size_t cache_row_num;
|
||||
size_t cache_size;
|
||||
|
||||
struct rhash_table *htable;
|
||||
//const struct ex_data_schema* ex_schema;
|
||||
int table_id;
|
||||
};
|
||||
|
||||
void cache_row_free(void*p)
|
||||
{
|
||||
free(*(char**)p);
|
||||
}
|
||||
|
||||
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
|
||||
|
||||
struct ex_data_runtime *ex_data_runtime_new(void (* data_free)(void *data))
|
||||
{
|
||||
struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1);
|
||||
|
||||
utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd);
|
||||
ex_data_rt->htable = rhash_table_create(data_free);
|
||||
}
|
||||
|
||||
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt)
|
||||
{
|
||||
if (ex_data_rt->cache_rows != NULL) {
|
||||
utarray_free(ex_data_rt->cache_rows);
|
||||
ex_data_rt->cache_rows=NULL;
|
||||
ex_data_rt->cache_row_num=0;
|
||||
}
|
||||
|
||||
if (ex_data_rt->htable != NULL) {
|
||||
rhash_table_destroy(ex_data_rt->htable);
|
||||
}
|
||||
|
||||
free(ex_data_rt);
|
||||
}
|
||||
104
src/maat_garbage_collection.cpp
Normal file
104
src/maat_garbage_collection.cpp
Normal file
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_garbage_collection.h
|
||||
* Description:
|
||||
* Authors: Zhengchao <zhengchao@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include "maat_utils.h"
|
||||
|
||||
#include <time.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/queue.h>
|
||||
#include <assert.h>
|
||||
|
||||
struct maat_garbage_bag {
|
||||
time_t create_time;
|
||||
int timeout;
|
||||
int ok_times;
|
||||
void *garbage;
|
||||
void (* garbage_free)(void *garbage);
|
||||
TAILQ_ENTRY(maat_garbage_bag) entries;
|
||||
};
|
||||
TAILQ_HEAD(maat_garbage_q, maat_garbage_bag);
|
||||
|
||||
struct maat_garbage_bin {
|
||||
struct maat_garbage_q garbage_q;
|
||||
size_t bag_cnt;
|
||||
int timeout_seconds;
|
||||
};
|
||||
|
||||
struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout)
|
||||
{
|
||||
struct maat_garbage_bin* bin = ALLOC(struct maat_garbage_bin, 1);
|
||||
|
||||
TAILQ_INIT(&bin->garbage_q);
|
||||
bin->timeout_seconds = default_timeout;
|
||||
|
||||
return bin;
|
||||
}
|
||||
|
||||
void maat_garbage_bin_free(struct maat_garbage_bin* bin)
|
||||
{
|
||||
struct maat_garbage_bag *p = NULL;
|
||||
|
||||
while (p = TAILQ_FIRST(&bin->garbage_q)) {
|
||||
p->garbage_free(p->garbage);
|
||||
TAILQ_REMOVE(&bin->garbage_q, p, entries);
|
||||
free(p);
|
||||
bin->bag_cnt--;
|
||||
}
|
||||
|
||||
free(bin);
|
||||
}
|
||||
|
||||
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin)
|
||||
{
|
||||
return bin->bag_cnt;
|
||||
}
|
||||
|
||||
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *))
|
||||
{
|
||||
struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1);
|
||||
|
||||
bag->create_time = time(NULL);
|
||||
bag->timeout = bin->timeout_seconds;
|
||||
bag->garbage = garbage;
|
||||
bag->garbage_free = func;
|
||||
TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
|
||||
bin->bag_cnt++;
|
||||
}
|
||||
void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
|
||||
{
|
||||
struct maat_garbage_bag *p = NULL, *tmp = NULL;
|
||||
size_t n_clollected = 0, n_bag = 0;
|
||||
time_t now = time(NULL);
|
||||
|
||||
for (p = TAILQ_FIRST(&bin->garbage_q); p != NULL; p = tmp) {
|
||||
tmp = TAILQ_NEXT(p, entries);
|
||||
if ((now - p->create_time) > p->timeout || p->timeout == 0) {
|
||||
p->garbage_free(p->garbage);
|
||||
TAILQ_REMOVE(&bin->garbage_q, p, entries);
|
||||
free(p);
|
||||
n_clollected++;
|
||||
}
|
||||
n_bag++;
|
||||
}
|
||||
|
||||
assert(bin->bag_cnt == n_bag);
|
||||
bin->bag_cnt -= n_clollected;
|
||||
}
|
||||
|
||||
void maat_garbage_collect_by_force(struct maat_garbage_bin* bin)
|
||||
{
|
||||
struct maat_garbage_bag *p = NULL;
|
||||
while (p = TAILQ_FIRST(&bin->garbage_q)) {
|
||||
p->garbage_free(p->garbage);
|
||||
TAILQ_REMOVE(&bin->garbage_q, p, entries);
|
||||
free(p);
|
||||
bin->bag_cnt--;
|
||||
}
|
||||
}
|
||||
133
src/maat_kv.cpp
Normal file
133
src/maat_kv.cpp
Normal file
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_kv_map.cpp
|
||||
* Description:
|
||||
* Authors: Zheng chao <zhengchao@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <ctype.h>
|
||||
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_utils.h"
|
||||
|
||||
#define MAAT_KV_MAX_KEY_LEN 512
|
||||
|
||||
struct maat_kv_pair
|
||||
{
|
||||
char* key; //must be lower case.
|
||||
size_t keylen;
|
||||
int val;
|
||||
UT_hash_handle hh;
|
||||
};
|
||||
|
||||
struct maat_kv_store
|
||||
{
|
||||
struct maat_kv_pair* hash;
|
||||
};
|
||||
|
||||
void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_sz)
|
||||
{
|
||||
for (size_t i = 0; i < src_len && i < dst_sz; i++) {
|
||||
dst[i] = tolower(src[i]);
|
||||
}
|
||||
}
|
||||
|
||||
struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, int value)
|
||||
{
|
||||
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
|
||||
|
||||
kv->key = ALLOC(char, keylen);
|
||||
kv->keylen = keylen;
|
||||
kv->val = value;
|
||||
|
||||
strlowercase(key, keylen, kv->key, kv->keylen);
|
||||
|
||||
return kv;
|
||||
}
|
||||
|
||||
void maat_kv_pair_free(struct maat_kv_pair* kv)
|
||||
{
|
||||
free(kv->key);
|
||||
kv->key = NULL;
|
||||
|
||||
free(kv);
|
||||
}
|
||||
|
||||
struct maat_kv_store* maat_kv_store_new(void)
|
||||
{
|
||||
struct maat_kv_store* store = ALLOC(struct maat_kv_store, 1);
|
||||
return store;
|
||||
}
|
||||
|
||||
void maat_kv_store_free(struct maat_kv_store* store)
|
||||
{
|
||||
if (NULL == store) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct maat_kv_pair *kv = NULL;
|
||||
struct maat_kv_pair *tmp_kv = NULL;
|
||||
|
||||
HASH_ITER(hh, store->hash, kv, tmp_kv) {
|
||||
HASH_DEL(store->hash, kv);
|
||||
maat_kv_pair_free(kv);
|
||||
}
|
||||
|
||||
free(store);
|
||||
}
|
||||
|
||||
int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value)
|
||||
{
|
||||
if (keylen > MAAT_KV_MAX_KEY_LEN) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct maat_kv_pair *kv = NULL;
|
||||
struct maat_kv_pair *tmp_kv = NULL;
|
||||
|
||||
kv = maat_kv_pair_new(key, keylen, value);
|
||||
HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv);
|
||||
if (tmp_kv) {
|
||||
maat_kv_pair_free(kv);
|
||||
return -1;
|
||||
}
|
||||
|
||||
HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int maat_kv_register(struct maat_kv_store* store, const char* key, int value)
|
||||
{
|
||||
int ret = 0;
|
||||
ret = maat_kv_register_unNull(store, key, strlen(key), value);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int* value)
|
||||
{
|
||||
struct maat_kv_pair *kv = NULL;
|
||||
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
|
||||
|
||||
if (keylen > MAAT_KV_MAX_KEY_LEN) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase));
|
||||
HASH_FIND(hh, store->hash, key_lowercase, keylen, kv);
|
||||
if (kv) {
|
||||
*value=kv->val;
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
int maat_kv_read(struct maat_kv_store * store, const char * key, int * value)
|
||||
{
|
||||
return maat_kv_read_unNull(store, key, strlen(key), value);
|
||||
}
|
||||
130
src/maat_rule.cpp
Normal file
130
src/maat_rule.cpp
Normal file
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_rule.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <pthread.h>
|
||||
#include <limits.h>
|
||||
|
||||
#include "maat_rule.h"
|
||||
#include "maat_config_monitor.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "maat_table_schema.h"
|
||||
|
||||
struct maat_runtime* create_maat_runtime(uint64_t version, struct maat *maat_instance)
|
||||
{
|
||||
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
|
||||
|
||||
maat_rt->version = version;
|
||||
maat_rt->table_rt_mgr = maat_table_runtime_manager_create(maat_instance->table_mgr);
|
||||
maat_rt->max_table_num = maat_table_manager_get_size(maat_instance->table_mgr);
|
||||
}
|
||||
|
||||
void update_maat_runtime(struct maat_runtime *maat_rt)
|
||||
{
|
||||
for (size_t i = 0; i < maat_rt->max_table_num; i++) {
|
||||
struct maat_table_runtime *table_rt = maat_table_runtime_get(maat_rt->table_rt_mgr, i);
|
||||
if (NULL == table_rt) {
|
||||
continue;
|
||||
}
|
||||
|
||||
enum maat_table_type table_type = maat_table_runtime_get_type(table_rt);
|
||||
}
|
||||
}
|
||||
|
||||
void destroy_maat_runtime(struct maat_runtime *maat_rt)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
void maat_start_cb(uint64_t new_version, int update_type, void *u_param)
|
||||
{
|
||||
struct maat *maat_instance = (struct maat *)u_param;
|
||||
|
||||
if (update_type == CONFIG_UPDATE_TYPE_FULL) {
|
||||
maat_instance->rebuilding_maat_rt = create_maat_runtime(new_version, maat_instance);
|
||||
} else {
|
||||
maat_instance->maat_version = new_version;
|
||||
}
|
||||
}
|
||||
|
||||
int maat_update_cb(const char* table_name, const char* line, void *u_param)
|
||||
{
|
||||
struct maat *maat_instance =(struct maat *)u_param;
|
||||
struct maat_runtime* maat_rt = NULL;
|
||||
int table_id = maat_table_manager_get_table_id(maat_instance->table_mgr, table_name);
|
||||
struct maat_table_schema* ptable = maat_table_manager_get_table_desc(maat_instance->table_mgr, table_id);
|
||||
|
||||
if (maat_instance->rebuilding_maat_rt != NULL) {
|
||||
maat_rt = maat_instance->rebuilding_maat_rt;
|
||||
} else {
|
||||
maat_rt = maat_instance->maat_rt;
|
||||
}
|
||||
|
||||
//TODO: update rule for table_schema
|
||||
update_maat_runtime();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void maat_finish_cb(void *u_param)
|
||||
{
|
||||
struct maat *maat_instance = (struct maat *)u_param;
|
||||
|
||||
if (maat_instance->latest_maat_rt != NULL) {
|
||||
update_maat_runtime(maat_instance->latest_maat_rt);
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
}
|
||||
|
||||
void *rule_monitor_loop(void *arg)
|
||||
{
|
||||
char err_str[NAME_MAX] = {0};
|
||||
struct maat *maat_instance = (struct maat *)arg;
|
||||
|
||||
while (maat_instance->is_running) {
|
||||
usleep(1000 * 1000);
|
||||
if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
|
||||
switch (maat_instance->rule_import_type) {
|
||||
case RULE_IMPORT_TYPE_IRIS:
|
||||
config_monitor_traverse(maat_instance->maat_version,
|
||||
maat_instance->iris_ctx.inc_dir,
|
||||
maat_start_cb,
|
||||
maat_update_cb,
|
||||
maat_finish_cb,
|
||||
maat_instance);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (maat_instance->latest_maat_rt != NULL) {
|
||||
struct maat_runtime *old_maat_rt = maat_instance->maat_rt;
|
||||
maat_instance->maat_rt = maat_instance->latest_maat_rt;
|
||||
|
||||
if (old_maat_rt != NULL) {
|
||||
|
||||
}
|
||||
|
||||
maat_instance->latest_maat_rt = NULL;
|
||||
maat_instance->maat_version = maat_instance->maat_rt->version;
|
||||
maat_instance->last_full_version = maat_instance->maat_rt->version;
|
||||
}
|
||||
|
||||
if (maat_instance->maat_rt != NULL) {
|
||||
update_maat_runtime(maat_instance->maat_rt);
|
||||
}
|
||||
|
||||
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
126
src/maat_table_runtime.cpp
Normal file
126
src/maat_table_runtime.cpp
Normal file
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_table_runtime.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <hs/hs.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "maat_table_runtime.h"
|
||||
#include "uthash/uthash.h"
|
||||
#include "maat_ex_data.h"
|
||||
#include "adapter_hs.h"
|
||||
|
||||
struct maat_expr_runtime {
|
||||
struct adapter_hs *hs;
|
||||
//rcu_hash *htable;
|
||||
};
|
||||
|
||||
struct maat_ip_plugin_runtime {
|
||||
//struct ip_matcher* ip_matcher;
|
||||
struct ex_data_runtime* ex_data_rt;
|
||||
};
|
||||
|
||||
struct maat_table_runtime {
|
||||
enum maat_table_type table_type; // table schema已有type??
|
||||
uint32_t rule_num;
|
||||
|
||||
union {
|
||||
struct maat_expr_runtime *expr_rt;
|
||||
struct maat_ip_plugin_runtime *ip_plugin_rt;
|
||||
};
|
||||
|
||||
//ex_data_rt
|
||||
//table相关指针
|
||||
};
|
||||
|
||||
struct maat_table_runtime_manager {
|
||||
struct maat_table_runtime **table_rt;
|
||||
size_t n_table_rt;
|
||||
};
|
||||
|
||||
struct maat_table_runtime *table_runtime_new(enum maat_table_type table_type, int max_thread_num, struct maat_garbage_bin* bin)
|
||||
{
|
||||
struct maat_table_runtime *table_rt = ALLOC(struct maat_table_runtime, 1);
|
||||
table_rt->table_type = table_type;
|
||||
switch (table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
table_rt->expr_rt->ex_data_rt = ex_data_runtime_new(NULL);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
table_rt->ip_plugin_rt->ex_data_rt = ex_data_runtime_new(NULL);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return table_rt;
|
||||
}
|
||||
|
||||
void table_runtime_free(struct maat_table_runtime * table_rt)
|
||||
{
|
||||
switch (table_rt->table_type)
|
||||
{
|
||||
case TABLE_TYPE_EXPR:
|
||||
ex_data_runtime_free(table_rt->expr_rt->ex_data_rt);
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ex_data_runtime_free(table_rt->ip_plugin_rt->ex_data_rt);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
free(table_rt);
|
||||
}
|
||||
|
||||
struct maat_table_runtime_manager *maat_table_runtime_manager_create(struct maat_table_manager *table_mgr, int max_thread_num, struct maat_garbage_bin* garbage_bin)
|
||||
{
|
||||
if (NULL == table_mgr) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct maat_table_runtime_manager *table_rt_mgr = ALLOC(struct maat_table_runtime_manager, 1);
|
||||
table_rt_mgr->n_table_rt = maat_table_manager_get_size(table_mgr);
|
||||
table_rt_mgr->table_rt = ALLOC(struct maat_table_runtime *, table_rt_mgr->n_table_rt);
|
||||
|
||||
for (size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
|
||||
enum maat_table_type table_type = maat_table_manager_get_table_type(table_mgr, i);
|
||||
if (table_type == TABLE_TYPE_MAX) {
|
||||
continue;
|
||||
}
|
||||
table_rt_mgr->table_rt[i] = table_runtime_new(table_type, max_thread_num, garbage_bin);
|
||||
}
|
||||
|
||||
return table_rt_mgr;
|
||||
}
|
||||
|
||||
void maat_table_runtime_manager_destroy(struct maat_table_runtime_manager *table_rt_mgr)
|
||||
{
|
||||
for(size_t i = 0; i < table_rt_mgr->n_table_rt; i++) {
|
||||
table_runtime_free(table_rt_mgr->table_rt[i]);
|
||||
table_rt_mgr->table_rt[i] = NULL;
|
||||
}
|
||||
|
||||
free(table_rt_mgr->table_rt);
|
||||
table_rt_mgr->table_rt = NULL;
|
||||
free(table_rt_mgr);
|
||||
}
|
||||
|
||||
struct maat_table_runtime *maat_table_runtime_get(struct maat_table_runtime_manager *table_rt_mgr, int table_id)
|
||||
{
|
||||
assert(table_id < (int)table_rt_mgr->n_table_rt);
|
||||
return table_rt_mgr->table_rt[table_id];
|
||||
}
|
||||
|
||||
enum maat_table_type maat_table_runtime_get_type(struct maat_table_runtime* table_rt)
|
||||
{
|
||||
return table_rt->table_type;
|
||||
}
|
||||
456
src/maat_table_schema.cpp
Normal file
456
src/maat_table_schema.cpp
Normal file
@@ -0,0 +1,456 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_table_schema.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "maat_table_schema.h"
|
||||
#include "maat_kv.h"
|
||||
#include "cJSON.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "maat_ex_data.h"
|
||||
|
||||
|
||||
#define MAX_TABLE_NUM 256
|
||||
#define MAX_TABLE_LINE_SIZE (1024 * 16)
|
||||
#define MAX_ITEM_STR 64
|
||||
|
||||
struct expr_item {
|
||||
int item_id;
|
||||
int group_id;
|
||||
char district[MAX_ITEM_STR]; //128
|
||||
char keywords[MAX_ITEM_STR]; //1k
|
||||
char expr_type[MAX_ITEM_STR];//enum
|
||||
char match_method[MAX_ITEM_STR]; //enum
|
||||
int is_hexbin;
|
||||
int case_sensitive;
|
||||
int is_valid;
|
||||
|
||||
//rule_tag; 只存在schema里
|
||||
//int have_exdata;
|
||||
//struct ex_data *ex_data; //hash表
|
||||
};
|
||||
|
||||
struct ip_plugin_item {
|
||||
int row_id;
|
||||
int ip_type;
|
||||
int start_ip;
|
||||
int end_ip;
|
||||
int valid_flag;
|
||||
int rule_tag;
|
||||
int have_exdata;
|
||||
struct ex_data *ex_data;
|
||||
};
|
||||
|
||||
struct maat_table_item {
|
||||
enum maat_table_type table_type;
|
||||
union {
|
||||
struct expr_item expr_item;
|
||||
struct ip_plugin_item ip_plugin_item;
|
||||
};
|
||||
};
|
||||
|
||||
struct expr_table_schema {
|
||||
int item_id_column;
|
||||
int group_id_column;
|
||||
int district_column;
|
||||
int keywords_column;
|
||||
int expr_type_column;
|
||||
int match_method_column;
|
||||
int is_hexbin_column;
|
||||
int case_sensitive_column;
|
||||
/* valid means add, invalid means delete */
|
||||
int is_valid_column;
|
||||
int have_exdata;
|
||||
struct ex_data_schema *ex_schema;
|
||||
};
|
||||
|
||||
struct ip_plugin_table_schema {
|
||||
int row_id_column;
|
||||
int ip_type_column;
|
||||
int start_ip_column;
|
||||
int end_ip_column;
|
||||
int valid_flag_column;
|
||||
int rule_tag_column;
|
||||
int have_exdata;
|
||||
struct ex_data_schema *ex_schema;
|
||||
};
|
||||
|
||||
struct maat_table_schema {
|
||||
int table_id;
|
||||
sds table_name;
|
||||
enum maat_table_type table_type;
|
||||
union {
|
||||
struct expr_table_schema expr;
|
||||
struct ip_plugin_table_schema ip_plugin;
|
||||
};
|
||||
};
|
||||
|
||||
struct maat_table_manager {
|
||||
struct maat_table_schema *schema_table[MAX_TABLE_NUM];
|
||||
size_t n_schema_table;
|
||||
|
||||
struct maat_kv_store *tablename2id_map;
|
||||
};
|
||||
|
||||
struct maat_table_schema *table_schema_new(void)
|
||||
{
|
||||
struct maat_table_schema *ptable = ALLOC(struct maat_table_schema, 1);
|
||||
|
||||
return ptable;
|
||||
}
|
||||
|
||||
void table_schema_free(struct maat_table_schema *ptable)
|
||||
{
|
||||
free(ptable);
|
||||
}
|
||||
|
||||
int read_expr_table_schema(cJSON *root, struct maat_table_schema *ptable)
|
||||
{
|
||||
int read_cnt = 0;
|
||||
cJSON *item = NULL;
|
||||
|
||||
item = cJSON_GetObjectItem(root, "table_id");
|
||||
if (item != NULL && item->type == cJSON_String) {
|
||||
ptable->table_id = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "table_name");
|
||||
if (item != NULL && item->type == cJSON_String) {
|
||||
memcpy(ptable->table_name, item->valuestring, strlen(item->valuestring));
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "item_id");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.item_id_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "group_id");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.group_id_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "district");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.district_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "keywords");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.keywords_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "expr_type");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.expr_type_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "match_method");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.match_method_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "is_hexbin");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.is_hexbin_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "case_sensitive");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.case_sensitive_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "is_valid");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->expr.is_valid_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
if (read_cnt < 10) {
|
||||
return -1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int read_ip_plugin_table_schema(cJSON *root, struct maat_table_schema *ptable)
|
||||
{
|
||||
int read_cnt = 0;
|
||||
cJSON *item = NULL;
|
||||
|
||||
item = cJSON_GetObjectItem(root, "table_id");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->table_id = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "table_name");
|
||||
if (item != NULL && item->type == cJSON_String) {
|
||||
memcpy(ptable->table_name, item->valuestring, strlen(item->valuestring));
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "row_id");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->ip_plugin.row_id_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "ip_type");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->ip_plugin.ip_type_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "start_ip");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->ip_plugin.start_ip_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "end_ip");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->ip_plugin.end_ip_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "valid");
|
||||
if (item != NULL && item->type == cJSON_Number) {
|
||||
ptable->ip_plugin.valid_flag_column = item->valueint;
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
if (read_cnt < 7) {
|
||||
return -1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int table_schema_populate(const char *line, struct maat_table_schema *ptable,
|
||||
struct maat_kv_store* reserved_word_map)
|
||||
{
|
||||
int ret = -1;
|
||||
int read_cnt = 0;
|
||||
cJSON *root = NULL;
|
||||
cJSON *item = NULL;
|
||||
sds copy_line = sdsnew(line);
|
||||
sds table_type_str = NULL;
|
||||
|
||||
root = cJSON_Parse(copy_line);
|
||||
if (NULL == root) {
|
||||
goto next;
|
||||
}
|
||||
|
||||
item = cJSON_GetObjectItem(root, "table_type");
|
||||
if (item != NULL && item->type == cJSON_String) {
|
||||
table_type_str = sdsnew(item->valuestring);
|
||||
read_cnt++;
|
||||
}
|
||||
|
||||
ret = maat_kv_read(reserved_word_map, table_type_str, (int*)&(ptable->table_type));
|
||||
if (ret < 0) {
|
||||
goto next;
|
||||
}
|
||||
|
||||
switch (ptable->table_type) {
|
||||
case TABLE_TYPE_EXPR:
|
||||
ret = read_expr_table_schema(root, ptable);
|
||||
if (ret < 0) {
|
||||
goto next;
|
||||
}
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
ret = read_ip_plugin_table_schema(root, ptable);
|
||||
if (ret < 0) {
|
||||
goto next;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
next:
|
||||
cJSON_Delete(root);
|
||||
sdsfree(copy_line);
|
||||
if (table_type_str != NULL) {
|
||||
sdsfree(table_type_str);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct maat_table_manager *maat_table_manager_create(sds table_info_path)
|
||||
{
|
||||
FILE *fp = fopen(table_info_path, "r");
|
||||
if (NULL == fp) {
|
||||
fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
|
||||
//log_error()
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct maat_kv_store* reserved_word_map = maat_kv_store_new();
|
||||
/* register table type reserved word */
|
||||
maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
|
||||
maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
|
||||
|
||||
int i = 0;
|
||||
char line[MAX_TABLE_LINE_SIZE] = {0};
|
||||
struct maat_table_manager *table_mgr = ALLOC(struct maat_table_manager, 1);
|
||||
struct maat_table_schema **pptable = table_mgr->schema_table;
|
||||
table_mgr->tablename2id_map = maat_kv_store_new();
|
||||
|
||||
while (NULL != fgets(line, sizeof(line), fp)) {
|
||||
i++;
|
||||
|
||||
if (line[0] == '#' || line[0] == ' ' || line[0] == '\t') {
|
||||
continue;
|
||||
}
|
||||
|
||||
struct maat_table_schema *table_schema = table_schema_new();
|
||||
int ret = table_schema_populate(line, table_schema, reserved_word_map);
|
||||
if (ret < 0) {
|
||||
fprintf(stderr, "Maat read table info %s line %d error: illegal table schema.\n",
|
||||
table_info_path, i);
|
||||
//log_error()
|
||||
goto error;
|
||||
}
|
||||
|
||||
if (table_schema->table_id >= MAX_TABLE_NUM) {
|
||||
fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",
|
||||
table_info_path, i, table_schema->table_id, MAX_TABLE_NUM);
|
||||
goto error;
|
||||
}
|
||||
|
||||
ret = maat_kv_register(table_mgr->tablename2id_map, table_schema->table_name, table_schema->table_id);
|
||||
if (ret < 0) {
|
||||
//log_error("Duplicate table %s of table id %d", table_schema->table_name, table_schema->table_id);
|
||||
goto error;
|
||||
}
|
||||
|
||||
pptable[table_schema->table_id] = table_schema;
|
||||
table_mgr->n_schema_table++;
|
||||
continue;
|
||||
error:
|
||||
table_schema_free(table_schema);
|
||||
}
|
||||
maat_kv_store_free(reserved_word_map);
|
||||
|
||||
return table_mgr;
|
||||
}
|
||||
|
||||
void maat_table_manager_destroy(struct maat_table_manager *table_mgr)
|
||||
{
|
||||
for (size_t i = 0; i < MAX_TABLE_NUM; i++) {
|
||||
if (NULL == table_mgr->schema_table[i]) {
|
||||
continue;
|
||||
}
|
||||
table_schema_free(table_mgr->schema_table[i]);
|
||||
table_mgr->schema_table[i] = NULL;
|
||||
}
|
||||
|
||||
maat_kv_store_free(table_mgr->tablename2id_map);
|
||||
free(table_mgr);
|
||||
}
|
||||
|
||||
int maat_table_manager_get_table_id(struct maat_table_manager* table_mgr, sds table_name)
|
||||
{
|
||||
int table_id = -1;
|
||||
|
||||
int ret=maat_kv_read(table_mgr->tablename2id_map, table_name, &table_id);
|
||||
if (ret > 0) {
|
||||
return table_id;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
enum maat_table_type maat_table_manager_get_table_type(struct maat_table_manager *table_mgr, int id)
|
||||
{
|
||||
if (table_mgr->schema_table[id] == NULL) {
|
||||
return TABLE_TYPE_MAX;
|
||||
}
|
||||
|
||||
return table_mgr->schema_table[id]->table_type;
|
||||
}
|
||||
|
||||
size_t maat_table_manager_get_size(struct maat_table_manager* table_mgr)
|
||||
{
|
||||
return table_mgr->n_schema_table;
|
||||
}
|
||||
|
||||
int populate_expr_table_item(sds line, struct expr_table_schema *expr_schema, struct expr_item *expr_item)
|
||||
{
|
||||
size_t column_offset = 0;
|
||||
size_t column_len = 0;
|
||||
|
||||
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
expr_item->item_id = atoi(line + column_offset);
|
||||
|
||||
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
|
||||
if (ret < 0) {
|
||||
return -1;
|
||||
}
|
||||
expr_item->group_id = atoi(line + column_offset);
|
||||
|
||||
}
|
||||
|
||||
int populate_ip_plugin_table_item(sds line, struct ip_plugin_table_schema *ip_plugin_schema, struct ip_plugin_item *ip_plugin_item)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct maat_table_item *
|
||||
maat_table_line_to_item(sds line, struct maat_table_schema *table_schema)
|
||||
{
|
||||
int ret = -1;
|
||||
struct maat_table_item *table_item = ALLOC(struct maat_table_item, 1);
|
||||
|
||||
switch (table_schema->table_type)
|
||||
{
|
||||
case TABLE_TYPE_EXPR:
|
||||
table_item->table_type = TABLE_TYPE_EXPR;
|
||||
ret = populate_expr_table_item(line, &table_schema->expr, &table_item->expr_item);
|
||||
if (ret < 0) {
|
||||
goto error;
|
||||
}
|
||||
break;
|
||||
case TABLE_TYPE_IP_PLUGIN:
|
||||
table_item->table_type = TABLE_TYPE_IP_PLUGIN;
|
||||
ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, &table_item->ip_plugin_item);
|
||||
if (ret < 0) {
|
||||
goto error;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return table_item;
|
||||
error:
|
||||
free(table_item);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
35
src/maat_utils.cpp
Normal file
35
src/maat_utils.cpp
Normal file
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_utils.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include "maat_utils.h"
|
||||
|
||||
int get_column_pos(sds line, int column_seq, size_t *offset, size_t *len)
|
||||
{
|
||||
const char* seps=" \t";
|
||||
char* saveptr=NULL, *subtoken=NULL, *str=NULL;
|
||||
sds dup_line = sdsdup(line);
|
||||
int i=0, ret=-1;
|
||||
for (str = dup_line; ; str = NULL)
|
||||
{
|
||||
subtoken = strtok_r(str, seps, &saveptr);
|
||||
if (subtoken == NULL)
|
||||
break;
|
||||
if(i==column_seq-1)
|
||||
{
|
||||
*offset=subtoken-dup_line;
|
||||
*len=strlen(subtoken);
|
||||
ret=0;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
free(dup_line);
|
||||
return ret;
|
||||
}
|
||||
227
src/rcu_hash.cpp
Normal file
227
src/rcu_hash.cpp
Normal file
@@ -0,0 +1,227 @@
|
||||
/*
|
||||
**********************************************************************************************
|
||||
* File: maat_rhash.cpp
|
||||
* Description:
|
||||
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
|
||||
* Date: 2022-10-31
|
||||
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
|
||||
***********************************************************************************************
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <pthread.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "rcu_hash.h"
|
||||
#include "maat_utils.h"
|
||||
#include "maat_garbage_collection.h"
|
||||
|
||||
#define GARBAGE_DEFAULT_TIMEOUT 60
|
||||
|
||||
struct rcu_hash_table {
|
||||
int is_updating;
|
||||
char effective_hash; // 'a' or 'b'
|
||||
|
||||
/* two hash map for rcu */
|
||||
struct rcu_hash_node *hashmap_a;
|
||||
struct rcu_hash_node *hashmap_b;
|
||||
|
||||
void (* data_free)(void *data);
|
||||
struct maat_garbage_bin *garbage_bin;
|
||||
|
||||
pthread_mutex_t update_mutex;
|
||||
};
|
||||
|
||||
struct rcu_hash_node {
|
||||
char *key;
|
||||
size_t key_len;
|
||||
void *data; //table_runtime解析成两个成员
|
||||
|
||||
UT_hash_handle hh_a;
|
||||
UT_hash_handle hh_b;
|
||||
};
|
||||
|
||||
void rcu_hash_node_free(struct rcu_hash_node *node, void (* data_free)(void *data))
|
||||
{
|
||||
if (node->key != NULL) {
|
||||
free(node->key);
|
||||
}
|
||||
|
||||
if (node->data != NULL) {
|
||||
data_free(node->data);
|
||||
}
|
||||
|
||||
free(node);
|
||||
}
|
||||
|
||||
struct rcu_hash_table *rcu_hash_new(void (* data_free)(void *data))
|
||||
{
|
||||
if (NULL == data_free) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct rcu_hash_table *htable = ALLOC(struct rcu_hash_table, 1);
|
||||
|
||||
htable->is_updating = 0;
|
||||
htable->effective_hash = 'a';
|
||||
htable->garbage_bin = maat_garbage_bin_new(GARBAGE_DEFAULT_TIMEOUT);
|
||||
htable->data_free = data_free;
|
||||
pthread_mutex_init(&htable->update_mutex, NULL);
|
||||
|
||||
return htable;
|
||||
}
|
||||
|
||||
void rcu_hash_free(struct rcu_hash_table *htable)
|
||||
{
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
struct rcu_hash_node *item = NULL;
|
||||
|
||||
if (htable != NULL) {
|
||||
HASH_ITER(hh_a, htable->hashmap_a, item, tmp) {
|
||||
HASH_DELETE(hh_a, htable->hashmap_a, item);
|
||||
rcu_hash_node_free(item, htable->data_free);
|
||||
}
|
||||
|
||||
HASH_ITER(hh_b, htable->hashmap_b, item, tmp) {
|
||||
HASH_DELETE(hh_b, htable->hashmap_b, item);
|
||||
rcu_hash_node_free(item, htable->data_free);
|
||||
}
|
||||
}
|
||||
|
||||
maat_garbage_bin_free(htable->garbage_bin);
|
||||
pthread_mutex_destroy(&htable->update_mutex);
|
||||
|
||||
free(htable);
|
||||
}
|
||||
|
||||
void rcu_hash_update_prepare(struct rcu_hash_table *htable)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
assert(htable->hashmap_b == NULL);
|
||||
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
|
||||
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, node->key, node->key_len, node);
|
||||
}
|
||||
} else {
|
||||
assert(htable->hashmap_a == NULL);
|
||||
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
|
||||
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, node->key, node->key_len, node);
|
||||
}
|
||||
}
|
||||
|
||||
htable->is_updating = 1;
|
||||
}
|
||||
|
||||
void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data)
|
||||
{
|
||||
struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1);
|
||||
memcpy(node->key, key, key_len);
|
||||
node->key_len = key_len;
|
||||
node->data = data;
|
||||
|
||||
if (!htable->is_updating) {
|
||||
rcu_hash_update_prepare(htable);
|
||||
}
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
if (NULL == node) {
|
||||
HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
}
|
||||
} else {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
if (NULL == node) {
|
||||
HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
|
||||
if (!htable->is_updating) {
|
||||
rcu_hash_update_prepare(htable);
|
||||
}
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
HASH_DELETE(hh_b, htable->hashmap_b, node);
|
||||
}
|
||||
} else {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
HASH_DELETE(hh_a, htable->hashmap_a, node);
|
||||
}
|
||||
}
|
||||
|
||||
if (node != NULL) {
|
||||
maat_garbage_bagging(htable->garbage_bin, node, (void (*)(void*))rcu_hash_node_free);
|
||||
}
|
||||
}
|
||||
|
||||
void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len)
|
||||
{
|
||||
struct rcu_hash_node *node = NULL;
|
||||
|
||||
if (htable->effective_hash == 'a') {
|
||||
HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
return node->data;
|
||||
}
|
||||
} else {
|
||||
HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node);
|
||||
if (node != NULL) {
|
||||
return node->data;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t rcu_hash_counts(struct rcu_hash_table *htable)
|
||||
{
|
||||
if (htable->effective_hash == 'a') {
|
||||
return HASH_CNT(hh_a, htable->hashmap_a);
|
||||
} else {
|
||||
return HASH_CNT(hh_b, htable->hashmap_b);
|
||||
}
|
||||
}
|
||||
|
||||
void rcu_hash_commit(struct rcu_hash_table *htable)
|
||||
{
|
||||
if (!htable->is_updating) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct rcu_hash_node *node = NULL;
|
||||
struct rcu_hash_node *tmp = NULL;
|
||||
|
||||
pthread_mutex_lock(&htable->update_mutex);
|
||||
if (!htable->is_updating) {
|
||||
pthread_mutex_unlock(&htable->update_mutex);
|
||||
return;
|
||||
}
|
||||
|
||||
/* updating hash_map is ready, so change effective hash_map */
|
||||
if (htable->effective_hash == 'a') {
|
||||
htable->effective_hash = 'b';
|
||||
usleep(100);
|
||||
HASH_ITER(hh_a, htable->hashmap_a, node, tmp) {
|
||||
HASH_DELETE(hh_a, htable->hashmap_a, node);
|
||||
}
|
||||
} else {
|
||||
htable->effective_hash = 'a';
|
||||
usleep(100);
|
||||
HASH_ITER(hh_b, htable->hashmap_b, node, tmp) {
|
||||
HASH_DELETE(hh_b, htable->hashmap_b, node);
|
||||
}
|
||||
}
|
||||
htable->is_updating = 0;
|
||||
//maat_garbage_collect_by_force(htable->garbage_bin);
|
||||
//rcu_garbage
|
||||
pthread_mutex_unlock(&htable->update_mutex);
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "../deps/uthash/uthash.h"
|
||||
|
||||
struct sub_string_pattern
|
||||
{
|
||||
int is_regex;
|
||||
size_t len;
|
||||
char *pattern;
|
||||
};
|
||||
struct string_pattern
|
||||
{
|
||||
unsigned int item_id;
|
||||
unsigned int group_id;
|
||||
char district[128];
|
||||
size_t n_sub_pattern;
|
||||
struct sub_string_pattern *sub_pattern;
|
||||
UT_hash_handle hh;
|
||||
};
|
||||
struct string_schema
|
||||
{
|
||||
int item_id_column;
|
||||
int district_column;
|
||||
int pattern_column;
|
||||
int is_regex_column;
|
||||
int valid_flag_column;
|
||||
int rule_tag_column;
|
||||
int have_exdata;
|
||||
struct EX_data_schema ex_schema;
|
||||
};
|
||||
struct string_runtime
|
||||
{
|
||||
struct hs_db_t *pure_literal_db;
|
||||
struct hs_db_t *regex_db;
|
||||
struct bool_matcher *bool_matcher;
|
||||
struct string_pattern *hash_effect_item;
|
||||
};
|
||||
@@ -1,15 +0,0 @@
|
||||
|
||||
struct ip_runtime
|
||||
{
|
||||
EX_data_rt *effect_items;
|
||||
struct ip_matcher *matcher;
|
||||
};
|
||||
struct table_runtime
|
||||
{
|
||||
struct hash_table item_hash;
|
||||
enum table_type type;
|
||||
union
|
||||
{
|
||||
struct ip_runtime *ip_rt;
|
||||
};
|
||||
};
|
||||
@@ -1,3 +1,11 @@
|
||||
add_executable(maat_api_gtest maat_api_gtest.cpp)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
|
||||
include_directories(${PROJECT_SOURCE_DIR}/deps)
|
||||
|
||||
add_executable(maat_api_gtest maat_api_gtest.cpp)
|
||||
target_link_libraries(maat_api_gtest maat_frame_static gtest_static)
|
||||
|
||||
add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
|
||||
target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
|
||||
|
||||
add_executable(rcu_hash_gtest rcu_hash_gtest.cpp)
|
||||
target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static)
|
||||
301
test/adapter_hs_gtest.cpp
Normal file
301
test/adapter_hs_gtest.cpp
Normal file
@@ -0,0 +1,301 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "maat_utils.h"
|
||||
#include "adapter_hs.h"
|
||||
#include "maat_table_schema.h"
|
||||
|
||||
int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr)
|
||||
{
|
||||
FILE *fp = fopen(filename, "r");
|
||||
|
||||
char line[4096] = {0};
|
||||
size_t i = 0;
|
||||
while (NULL != fgets(line, sizeof(line), fp)) {
|
||||
if (line[0] == '#' || line[0] == ' ' || line[0] == '\t') {
|
||||
continue;
|
||||
}
|
||||
|
||||
char pattern_buf[1024] = {0};
|
||||
int ret = sscanf(line, "%u\t%lu\t%s", &(expr[i].expr_id), &(expr[i].n_patterns), pattern_buf);
|
||||
EXPECT_NE(ret, 0);
|
||||
|
||||
char *expr_token = NULL;
|
||||
char *sub_expr_token = NULL;
|
||||
char *save_expr_ptr = NULL;
|
||||
size_t j = 0;
|
||||
for (expr_token = pattern_buf; ; expr_token = NULL) {
|
||||
sub_expr_token = strtok_r(expr_token, "&", &save_expr_ptr);
|
||||
if (sub_expr_token == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
char *pattern_token = sub_expr_token;
|
||||
char *save_pattern_ptr = NULL;
|
||||
char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr);
|
||||
expr[i].patterns[j].type = atoi(sub_pattern_token);
|
||||
size_t str_len = strlen(save_pattern_ptr);
|
||||
expr[i].patterns[j].pat = ALLOC(char, str_len);
|
||||
memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len);
|
||||
expr[i].patterns[j].pat_len = str_len;
|
||||
j++;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
*n_expr = i;
|
||||
fclose(fp);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void expr_array_free(and_expr_t expr_array[], size_t n_expr_array)
|
||||
{
|
||||
for (size_t i = 0; i < n_expr_array; i++) {
|
||||
for (size_t j = 0; j < expr_array[i].n_patterns; j++) {
|
||||
free(expr_array[i].patterns[j].pat);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(block_mode_initialize, invalid_input_parameter)
|
||||
{
|
||||
struct adapter_hs *hs_instance = NULL;
|
||||
and_expr_t exprs[5];
|
||||
|
||||
/* case1: invalid scan_mode parameter */
|
||||
hs_instance = adapter_hs_initialize(0, 1, exprs, 1);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
|
||||
/* case2: invalid expr parameter */
|
||||
hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 1);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
|
||||
/* case3: invalid expr num */
|
||||
hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, exprs, 0);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, invalid_input_parameter)
|
||||
{
|
||||
and_expr_t expr_array[64];
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 0);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
|
||||
hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
|
||||
n_expr_array = 1;
|
||||
expr_array[0].expr_id = 101;
|
||||
expr_array[0].n_patterns = 10;
|
||||
hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
|
||||
memset(expr_array, 0, sizeof(expr_array));
|
||||
n_expr_array = 1;
|
||||
expr_array[0].expr_id = 101;
|
||||
expr_array[0].n_patterns = 1;
|
||||
expr_array[0].patterns[0].type = 0;
|
||||
hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_EQ(hs_instance, nullptr);
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, hit_one_expr)
|
||||
{
|
||||
and_expr_t expr_array[64] = {0};
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_expr_array, 6);
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_NE(hs_instance, nullptr);
|
||||
expr_array_free(expr_array, n_expr_array);
|
||||
|
||||
char data0[64] = "luis";
|
||||
int result0[64] = {0};
|
||||
size_t n_result0 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result0, 0);
|
||||
|
||||
char data1[64] = "hello";
|
||||
int result1[64] = {0};
|
||||
size_t n_result1 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result1, 1);
|
||||
EXPECT_EQ(result1[0], 101);
|
||||
|
||||
char data2[64] = "world";
|
||||
int result2[64] = {0};
|
||||
size_t n_result2 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data2, strlen(data2), result2, &n_result2);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result2, 1);
|
||||
EXPECT_EQ(result2[0], 102);
|
||||
|
||||
adapter_hs_destroy(hs_instance);
|
||||
hs_instance = nullptr;
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, hit_two_expr)
|
||||
{
|
||||
and_expr_t expr_array[64] = {0};
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_expr_array, 6);
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_NE(hs_instance, nullptr);
|
||||
expr_array_free(expr_array, n_expr_array);
|
||||
|
||||
char data0[64] = "hello maat";
|
||||
int result0[64] = {0};
|
||||
size_t n_result0 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result0, 2);
|
||||
EXPECT_EQ(result0[0], 103);
|
||||
EXPECT_EQ(result0[1], 101);
|
||||
|
||||
char data1[64] = "maat World";
|
||||
int result1[64] = {0};
|
||||
size_t n_result1 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result1, 2);
|
||||
EXPECT_EQ(result1[0], 103);
|
||||
EXPECT_EQ(result1[1], 102);
|
||||
|
||||
adapter_hs_destroy(hs_instance);
|
||||
hs_instance = nullptr;
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, hit_three_expr)
|
||||
{
|
||||
and_expr_t expr_array[64] = {0};
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_expr_array, 6);
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_NE(hs_instance, nullptr);
|
||||
expr_array_free(expr_array, n_expr_array);
|
||||
|
||||
char data0[64] = "hello world";
|
||||
int result0[64] = {0};
|
||||
size_t n_result0 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result0, 3);
|
||||
EXPECT_EQ(result0[0], 104);
|
||||
EXPECT_EQ(result0[1], 102);
|
||||
EXPECT_EQ(result0[2], 101);
|
||||
|
||||
char data1[64] = "hello World";
|
||||
int result1[64] = {0};
|
||||
size_t n_result1 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result1, 3);
|
||||
EXPECT_EQ(result1[0], 104);
|
||||
EXPECT_EQ(result1[1], 102);
|
||||
EXPECT_EQ(result1[2], 101);
|
||||
|
||||
adapter_hs_destroy(hs_instance);
|
||||
hs_instance = nullptr;
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, hit_four_expr)
|
||||
{
|
||||
and_expr_t expr_array[64] = {0};
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_expr_array, 6);
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_NE(hs_instance, nullptr);
|
||||
expr_array_free(expr_array, n_expr_array);
|
||||
|
||||
char data0[64] = "1hello world";
|
||||
int result0[64] = {0};
|
||||
size_t n_result0 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result0, 4);
|
||||
EXPECT_EQ(result0[0], 105);
|
||||
EXPECT_EQ(result0[1], 104);
|
||||
EXPECT_EQ(result0[2], 102);
|
||||
EXPECT_EQ(result0[3], 101);
|
||||
|
||||
char data1[64] = "8hello World";
|
||||
int result1[64] = {0};
|
||||
size_t n_result1 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result1, 4);
|
||||
EXPECT_EQ(result1[0], 105);
|
||||
EXPECT_EQ(result1[1], 104);
|
||||
EXPECT_EQ(result1[2], 102);
|
||||
EXPECT_EQ(result1[3], 101);
|
||||
|
||||
adapter_hs_destroy(hs_instance);
|
||||
hs_instance = nullptr;
|
||||
}
|
||||
|
||||
TEST(block_mode_scan, hit_five_expr)
|
||||
{
|
||||
and_expr_t expr_array[64] = {0};
|
||||
size_t n_expr_array = 0;
|
||||
|
||||
int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_expr_array, 6);
|
||||
|
||||
struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
|
||||
EXPECT_NE(hs_instance, nullptr);
|
||||
expr_array_free(expr_array, n_expr_array);
|
||||
|
||||
char data0[64] = "1hello 2world";
|
||||
int result0[64] = {0};
|
||||
size_t n_result0 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result0, 5);
|
||||
EXPECT_EQ(result0[0], 106);
|
||||
EXPECT_EQ(result0[1], 105);
|
||||
EXPECT_EQ(result0[2], 104);
|
||||
EXPECT_EQ(result0[3], 102);
|
||||
EXPECT_EQ(result0[4], 101);
|
||||
|
||||
char data1[64] = "8hello 9World";
|
||||
int result1[64] = {0};
|
||||
size_t n_result1 = 0;
|
||||
ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1);
|
||||
EXPECT_EQ(ret, 0);
|
||||
EXPECT_EQ(n_result1, 5);
|
||||
EXPECT_EQ(result1[0], 106);
|
||||
EXPECT_EQ(result1[1], 105);
|
||||
EXPECT_EQ(result1[2], 104);
|
||||
EXPECT_EQ(result1[3], 102);
|
||||
EXPECT_EQ(result1[4], 101);
|
||||
|
||||
adapter_hs_destroy(hs_instance);
|
||||
hs_instance = nullptr;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int ret = 0;
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
ret = RUN_ALL_TESTS();
|
||||
return ret;
|
||||
}
|
||||
10
test/and_expr.conf
Normal file
10
test/and_expr.conf
Normal file
@@ -0,0 +1,10 @@
|
||||
# logic AND expressions sample
|
||||
# expr_id pattern_num expression(several patterns)
|
||||
# pattern_type:pattern_string&pattern_type:pattern_string&...
|
||||
# PATTERN_TYPE_STR(1) PATTERN_TYPE_REG(2)
|
||||
101 1 1:hello
|
||||
102 1 2:[W|w]orld
|
||||
103 1 1:maat
|
||||
104 2 1:hello&1:world
|
||||
105 2 2:[0-9]hello&1:world
|
||||
106 2 2:[0-9]hello&2:[0-9]world
|
||||
@@ -7,7 +7,7 @@ TEST(EQ_Test, Always_True) {
|
||||
}
|
||||
|
||||
TEST(maat_api, maat_scan_string) {
|
||||
int ret = maat_scan_string(nullptr, 0, nullptr, nullptr, 0);
|
||||
int ret = maat_scan_string(nullptr, 0, 0, nullptr, 0, nullptr, nullptr, nullptr);
|
||||
EXPECT_EQ(ret, 0);
|
||||
}
|
||||
|
||||
|
||||
50
test/rcu_hash_gtest.cpp
Normal file
50
test/rcu_hash_gtest.cpp
Normal file
@@ -0,0 +1,50 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "rcu_hash.h"
|
||||
#include "maat_utils.h"
|
||||
|
||||
struct user_data {
|
||||
int id;
|
||||
char name[32];
|
||||
};
|
||||
|
||||
void data_free(void *data)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
TEST(rcu_hash_new, invalid_input_parameter) {
|
||||
struct rcu_hash_table *htable = rcu_hash_new(nullptr);
|
||||
EXPECT_EQ(htable, nullptr);
|
||||
}
|
||||
|
||||
TEST(rcu_hash_add, one) {
|
||||
struct rcu_hash_table *htable = rcu_hash_new(data_free);
|
||||
EXPECT_NE(htable, nullptr);
|
||||
|
||||
struct user_data *data = ALLOC(struct user_data, 1);
|
||||
data->id = 101;
|
||||
char *name = "www.baidu.com";
|
||||
memcpy(data->name, name, strlen(name));
|
||||
char *key = "http_url";
|
||||
size_t key_len = strlen(key);
|
||||
|
||||
rcu_hash_add(htable, key, key_len, (void *)data);
|
||||
|
||||
void *res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_EQ(res, nullptr);
|
||||
|
||||
rcu_hash_commit(htable);
|
||||
|
||||
res = rcu_hash_find(htable, key, key_len);
|
||||
EXPECT_NE(res, nullptr);
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
int ret=0;
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
ret=RUN_ALL_TESTS();
|
||||
return ret;
|
||||
}
|
||||
31
test/table_info.conf
Normal file
31
test/table_info.conf
Normal file
@@ -0,0 +1,31 @@
|
||||
#each column seperated by '\t'
|
||||
[
|
||||
{
|
||||
"table_id":1,
|
||||
"table_name":"HTTP_URL",
|
||||
"table_type":"expr",
|
||||
"item_id":1,
|
||||
"group_id":2,
|
||||
"rule": {
|
||||
"keywords":3,
|
||||
"expr_type":4,
|
||||
"match_method":5,
|
||||
"is_hexbin":6,
|
||||
"case_sensitive":7,
|
||||
"is_valid":8
|
||||
}
|
||||
},
|
||||
{
|
||||
"table_id":2,
|
||||
"table_name":"TEST_IP_PLUGIN",
|
||||
"table_type":"ip_plugin",
|
||||
"item_id":1,
|
||||
"group_id":2,
|
||||
"rule": {
|
||||
"ip_type":2,
|
||||
"start_ip":3,
|
||||
"end_ip":4,
|
||||
"is_valid":5
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user