diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index f0519b3..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,45 +0,0 @@
-# Created by https://www.gitignore.io/api/c++,clion
-
-### C++ ###
-# Prerequisites
-*.d
-
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-*.smod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app
-
-# CMake
-cmake-build-*/
-build/*
-
-# Clion
-.idea/
-
-# Vscode
-.vscode/*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 1c2b299..0000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,224 +0,0 @@
-variables:
- GIT_STRATEGY: "clone"
- BUILD_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master"
- BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux"
- BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
- INSTALL_DEPENDENCY_LIBRARY: libMESA_handle_logger-devel libMESA_htable-devel pcre-devel librulescan-devel libMESA_field_stat2-devel libMESA_field_stat-devel sapp-devel framework_env openssl-devel libasan
-
-stages:
-- build
-
-.build_before_script:
- before_script:
- - mkdir -p $BUILD_PADDING_PREFIX/$CI_PROJECT_NAMESPACE/
- - ln -s $CI_PROJECT_DIR $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
- - cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
- - chmod +x ./ci/travis.sh
- - yum makecache
- - yum install -y elfutils-libelf-devel
-
-
-.build_by_travis_for_centos7:
- stage: build
- image: $BUILD_IMAGE_CENTOS7
- extends: .build_before_script
- script:
- - yum install -y libmnl-devel
- - yum install -y libnfnetlink-devel
- - ./ci/travis.sh
- - cd build
- tags:
- - share
-
-.build_by_travis_for_centos8:
- stage: build
- image: $BUILD_IMAGE_CENTOS8
- extends: .build_before_script
- script:
- - dnf --enablerepo=powertools install -y libmnl-devel
- - dnf --enablerepo=powertools install -y libnfnetlink-devel
- - ./ci/travis.sh
- tags:
- - share
-
-branch_build_debug_for_centos7:
- stage: build
- extends: .build_by_travis_for_centos7
- variables:
- BUILD_TYPE: Debug
- except:
- - /^develop.*$/i
- - /^master.*$/i
- - tags
-
-branch_build_release_for_centos7:
- stage: build
- variables:
- BUILD_TYPE: RelWithDebInfo
- extends: .build_by_travis_for_centos7
- except:
- - /^develop.*$/i
- - /^master.*$/i
- - tags
-
-develop_build_debug_for_centos7:
- stage: build
- extends: .build_by_travis_for_centos7
- variables:
- BUILD_TYPE: Debug
- PACKAGE: 1
- UPLOAD: 1
- ASAN_OPTION: ADDRESS
- TESTING_VERSION_BUILD: 1
- PULP3_REPO_NAME: framework-testing-x86_64.el7
- PULP3_DIST_NAME: framework-testing-x86_64.el7
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-debug"
- paths:
- - build/*.rpm
- only:
- - /^develop.*$/i
- - /^master.*$/i
-
-develop_build_release_for_centos7:
- stage: build
- extends: .build_by_travis_for_centos7
- variables:
- BUILD_TYPE: RelWithDebInfo
- PACKAGE: 1
- UPLOAD: 1
- TESTING_VERSION_BUILD: 1
- PULP3_REPO_NAME: framework-testing-x86_64.el7
- PULP3_DIST_NAME: framework-testing-x86_64.el7
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-release"
- paths:
- - build/*.rpm
- only:
- - /^develop.*$/i
- - /^master.*$/i
-
-release_build_debug_for_centos7:
- stage: build
- variables:
- BUILD_TYPE: Debug
- PACKAGE: 1
- UPLOAD: 1
- PULP3_REPO_NAME: framework-stable-x86_64.el7
- PULP3_DIST_NAME: framework-stable-x86_64.el7
- extends: .build_by_travis_for_centos7
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-debug"
- paths:
- - build/*.rpm
- only:
- - tags
-
-release_build_release_for_centos7:
- stage: build
- variables:
- BUILD_TYPE: RelWithDebInfo
- PACKAGE: 1
- UPLOAD: 1
- UPLOAD_SYMBOL_FILES: 1
- SYMBOL_TARGET: libmaatframe
- PULP3_REPO_NAME: framework-stable-x86_64.el7
- PULP3_DIST_NAME: framework-stable-x86_64.el7
- extends: .build_by_travis_for_centos7
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-release"
- paths:
- - build/*.rpm
- only:
- - tags
-
-branch_build_debug_for_centos8:
- stage: build
- extends: .build_by_travis_for_centos8
- variables:
- BUILD_TYPE: Debug
- except:
- - /^develop.*$/i
- - /^master.*$/i
- - tags
-
-branch_build_release_for_centos8:
- stage: build
- variables:
- BUILD_TYPE: RelWithDebInfo
- extends: .build_by_travis_for_centos8
- except:
- - /^develop.*$/i
- - /^master.*$/i
- - tags
-
-develop_build_debug_for_centos8:
- stage: build
- extends: .build_by_travis_for_centos8
- variables:
- BUILD_TYPE: Debug
- PACKAGE: 1
- UPLOAD: 1
- ASAN_OPTION: ADDRESS
- TESTING_VERSION_BUILD: 1
- PULP3_REPO_NAME: framework-testing-x86_64.el8
- PULP3_DIST_NAME: framework-testing-x86_64.el8
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-debug"
- paths:
- - build/*.rpm
- only:
- - /^develop.*$/i
- - /^master.*$/i
-
-develop_build_release_for_centos8:
- stage: build
- extends: .build_by_travis_for_centos8
- variables:
- BUILD_TYPE: RelWithDebInfo
- PACKAGE: 1
- UPLOAD: 1
- TESTING_VERSION_BUILD: 1
- PULP3_REPO_NAME: framework-testing-x86_64.el8
- PULP3_DIST_NAME: framework-testing-x86_64.el8
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-release"
- paths:
- - build/*.rpm
- only:
- - /^develop.*$/i
- - /^master.*$/i
-
-release_build_debug_for_centos8:
- stage: build
- variables:
- BUILD_TYPE: Debug
- PACKAGE: 1
- UPLOAD: 1
- PULP3_REPO_NAME: framework-stable-x86_64.el8
- PULP3_DIST_NAME: framework-stable-x86_64.el8
- extends: .build_by_travis_for_centos8
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-debug"
- paths:
- - build/*.rpm
- only:
- - tags
-
-release_build_release_for_centos8:
- stage: build
- variables:
- BUILD_TYPE: RelWithDebInfo
- PACKAGE: 1
- UPLOAD: 1
- UPLOAD_SYMBOL_FILES: 1
- SYMBOL_TARGET: libmaatframe
- PULP3_REPO_NAME: framework-stable-x86_64.el8
- PULP3_DIST_NAME: framework-stable-x86_64.el8
- extends: .build_by_travis_for_centos8
- artifacts:
- name: "maatframe-$CI_COMMIT_REF_NAME-release"
- paths:
- - build/*.rpm
- only:
- - tags
diff --git a/CMakeLists.txt b/CMakeLists.txt
deleted file mode 100644
index 28e9268..0000000
--- a/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-cmake_minimum_required (VERSION 2.8)
-set(lib_name maatframe)
-project (maatframe)
-
-set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-include(Version)
-
-SET(CMAKE_INSTALL_PREFIX /opt/MESA/)
-
-set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall)
-set(MAAT_DEPEND_DYN_LIB MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto z)
-include_directories(${PROJECT_SOURCE_DIR}/inc/)
-include_directories(/opt/MESA/include/)
-
-#for ASAN
-set(ASAN_OPTION "OFF" CACHE STRING " set asan type chosen by the user, using OFF as default")
-set_property(CACHE ASAN_OPTION PROPERTY STRINGS OFF ADDRESS THREAD)
-message(STATUS "ASAN_OPTION='${ASAN_OPTION}'")
-
-if(ASAN_OPTION MATCHES "ADDRESS")
- set(CMAKE_C_FLAGS "${CMAKADDRESS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer")
- set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
- set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
-elseif(ASAN_OPTION MATCHES "THREAD")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer")
- set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
- set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
-endif()
-# end of for ASAN
-
-enable_testing()
-
-add_subdirectory (vendor)
-add_subdirectory (src)
-add_subdirectory (test)
-add_subdirectory (tools)
-
-include(Package)
diff --git a/Makefile b/Makefile
deleted file mode 100644
index ac23531..0000000
--- a/Makefile
+++ /dev/null
@@ -1,41 +0,0 @@
-BUILD_DIR = $(CURDIR)/build
-LOCAL_DIR = $(CURDIR)
-DEBUG_FLAGS = -DCMAKE_BUILD_TYPE=Debug
-REL_FLAGS = -DCMAKE_BUILD_TYPE=RelWithDebInfo
-
-ifneq ($(INSTALL_PREFIX),)
-DEBUG_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX)
-REL_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX)
-endif
-
-all: _make_build_dir _compile_rel
-
-PHONY: all _make_build_dir _compile_debug _compile_rel _install \
- build_release build_debug install
-
-_make_build_dir:
- mkdir -p $(BUILD_DIR)
-
-_compile_debug:
- cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(DEBUG_FLAGS) && make
-
-_compile_rel:
- cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(REL_FLAGS) && make
-
-_install:
- cd $(BUILD_DIR) && make install
-_package:
- cd $(BUILD_DIR) && make package
-_clean:
- rm -rf $(BUILD_DIR)
-
-# Release Version, No Debug Symbol and Optimized with -O2
-release: _make_build_dir _compile_rel
-# Debug Version, Optimized with -O0
-debug: _make_build_dir _compile_debug
-# Install
-install: _install
-# Package
-package: _package
-# Clean
-clean: _clean
diff --git a/ancient_history.txt b/ancient_history.txt
deleted file mode 100644
index 6076da0..0000000
--- a/ancient_history.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-Author:Zheng Chao zhengchao@iie.ac.cn
-2014-04-01 create this project;
-2014-08-19 first online version;
-2014-09-09 support expr offset;
-2014-09-26 add group function;
-2014-12-03 support regex grouping;
-2014-12-17 write some comments in Maat_rule.h;
-2014-12-22 add group_num in compile config table;
-2015-01-04 make string match not case sensitive,fix garbage_bagging malloc wrong size bug;
-2015-01-07 replace assert of database rule format,unescape '&' of EXPR_TYPE_OFFSET;
-2015-01-22 ouput IRIS index file format error;verify string scan input data
-and data len;
-2015-01-28 enfore parameter check in Maat_stream_scan_string_detail to
-optimize performance;
-2015-02-04 support stream scan data with offset;
-2015-02-20 add JSON config mode and add a demo;
-2015-03-02 when use iconv_convert from gbk to big5,automaticly alternate gbk to
-gb2312;
-2015-04-13 1)Maat_xx_scan will return 0 if table has no config instead of -1;
- 2)use my_scandir as replacement of glibc's scandir to adapt dictator
- malloc wrap;
- 3)if iconv take no effect,jump over this string;
- 4)compile shortcut for performance;
- 5)verify if region id is unique;
-2015-04-20 fix Maat_stream_scan_string_detail and Maat_stream_scan_string wrong
- data len when open cross caching;
-2015-04-21 fix add_group_to_compile bug on compatible to none-group mode;
-2015-04-29 fix false hit compile rule of one more region bug;
-2015-05-06 fix add_group_to_compile return add failed status bug;
-2015-05-07 1)use rwlock instead of mutex in _mi_rule;
- 2)fix inc postpone update bug;
-2015-05-21 1)addapt rulescan ip addr host order requirement;
- 2) use readdir_r in my_scandir;
- 3) fix segmentfault when have no config,again;
- 4) maat_json support plugin table;
-2015-07-03 1)print error when Maat_summon_feather with a dir has no valid
-indexfile;
- 2)check AND_EXPR's match method;
- 3)remove restriction of IP table's protocol;
- 4)fix invalid write in insert_set_id function;
-2015-07-06 1)handle wrong expr format like "aa&&bb" and "aa&bb&";
- 2)iconv_convert performance optimized;
-2015-10-19 check table_type in callback register;
-2015-11-09 1)add digest feature;
- 2) split some code from Maat_rule.cpp to Maat_api.cpp;
-2015-12-24 change plugin table update mechanism to save memory;
-2016-01-20 maat_finish_cb adapt empty inc callback on a NULL scanner;
-2016-01-31 trigger plugin table's callback ONLY on its table's changed;
diff --git a/autorevision.sh b/autorevision.sh
deleted file mode 100644
index 3baa179..0000000
--- a/autorevision.sh
+++ /dev/null
@@ -1,1268 +0,0 @@
-#!/bin/sh
-
-# Copyright (c) 2012 - 2016 dak180 and contributors. See
-# https://opensource.org/licenses/mit-license.php or the included
-# COPYING.md for licence terms.
-#
-# autorevision - extracts metadata about the head version from your
-# repository.
-
-# Usage message.
-arUsage() {
- cat > "/dev/stderr" << EOF
-usage: autorevision {-t output-type | -s symbol} [-o cache-file [-f] ] [-V]
- Options include:
- -t output-type = specify output type
- -s symbol = specify symbol output
- -o cache-file = specify cache file location
- -f = force the use of cache data
- -U = check for untracked files in svn
- -V = emit version and exit
- -? = help message
-
-The following are valid output types:
- clojure = clojure file
- c = C/C++ file
- h = Header for use with c/c++
- hpp = Alternate C++ header strings with namespace
- ini = INI file
- java = Java file
- javaprop = Java properties file
- js = javascript file
- json = JSON file
- lua = Lua file
- m4 = m4 file
- matlab = matlab file
- octave = octave file
- php = PHP file
- pl = Perl file
- py = Python file
- rpm = rpm file
- scheme = scheme file
- sh = Bash sytax
- swift = Swift file
- tex = (La)TeX file
- xcode = Header useful for populating info.plist files
- cmake = CMake file
-
-
-The following are valid symbols:
- VCS_TYPE
- VCS_BASENAME
- VCS_UUID
- VCS_NUM
- VCS_DATE
- VCS_BRANCH
- VCS_TAG
- VCS_TICK
- VCS_EXTRA
- VCS_FULL_HASH
- VCS_SHORT_HASH
- VCS_WC_MODIFIED
- VCS_ACTION_STAMP
-EOF
- exit 1
-}
-
-# Config
-ARVERSION="&&ARVERSION&&"
-TARGETFILE="/dev/stdout"
-while getopts ":t:o:s:VfU" OPTION; do
- case "${OPTION}" in
- t)
- AFILETYPE="${OPTARG}"
- ;;
- o)
- CACHEFILE="${OPTARG}"
- ;;
- f)
- CACHEFORCE="1"
- ;;
- s)
- VAROUT="${OPTARG}"
- ;;
- U)
- UNTRACKEDFILES="1"
- ;;
- V)
- echo "autorevision ${ARVERSION}"
- exit 0
- ;;
- ?)
- # If an unknown flag is used (or -?):
- arUsage
- ;;
- esac
-done
-
-if [ ! -z "${VAROUT}" ] && [ ! -z "${AFILETYPE}" ]; then
- # If both -s and -t are specified:
- echo "error: Improper argument combination." 1>&2
- exit 1
-elif [ -z "${VAROUT}" ] && [ -z "${AFILETYPE}" ]; then
- # If neither -s or -t are specified:
- arUsage
-elif [ -z "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then
- # If -f is specified without -o:
- arUsage
-elif [ ! -f "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then
- # If we are forced to use the cache but it does not exist.
- echo "error: Cache forced but no cache found." 1>&2
- exit 1
-fi
-
-# Make sure that the path we are given is one we can source
-# (dash, we are looking at you).
-if [ ! -z "${CACHEFILE}" ] && ! echo "${CACHEFILE}" | grep -q '^\.*/'; then
- CACHEFILE="./${CACHEFILE}"
-fi
-
-GENERATED_HEADER="Generated by autorevision - do not hand-hack!"
-
-# Functions to extract data from different repo types.
-# For git repos
-# shellcheck disable=SC2039,SC2164,SC2155
-gitRepo() {
- local oldPath="${PWD}"
-
- cd "$(git rev-parse --show-toplevel)"
-
- VCS_TYPE="git"
-
- VCS_BASENAME="$(basename "${PWD}")"
-
- VCS_UUID="$(git rev-list --max-parents=0 --date-order --reverse HEAD 2>/dev/null | sed -n 1p)"
- if [ -z "${VCS_UUID}" ]; then
- VCS_UUID="$(git rev-list --topo-order HEAD | tail -n 1)"
- fi
-
- # Is the working copy clean?
- test -z "$(git status --untracked-files=normal --porcelain)"
- VCS_WC_MODIFIED="${?}"
-
- # Enumeration of changesets
- VCS_NUM="$(git rev-list --count HEAD 2>/dev/null)"
- if [ -z "${VCS_NUM}" ]; then
- echo "warning: Counting the number of revisions may be slower due to an outdated git version less than 1.7.2.3. If something breaks, please update it." 1>&2
- VCS_NUM="$(git rev-list HEAD | wc -l)"
- fi
-
- # This may be a git-svn remote. If so, report the Subversion revision.
- if [ -z "$(git config svn-remote.svn.url 2>/dev/null)" ]; then
- # The full revision hash
- VCS_FULL_HASH="$(git rev-parse HEAD)"
-
- # The short hash
- VCS_SHORT_HASH="$(echo "${VCS_FULL_HASH}" | cut -b 1-7)"
- else
- # The git-svn revision number
- VCS_FULL_HASH="$(git svn find-rev HEAD)"
- VCS_SHORT_HASH="${VCS_FULL_HASH}"
- fi
-
- # Current branch
- VCS_BRANCH="$(git rev-parse --symbolic-full-name --verify "$(git name-rev --name-only --no-undefined HEAD 2>/dev/null)" 2>/dev/null | sed -e 's:refs/heads/::' | sed -e 's:refs/::')"
-
- # Cache the description
- local DESCRIPTION="$(git describe --long --tags 2>/dev/null)"
-
- # Current or last tag ancestor (empty if no tags)
- VCS_TAG="$(echo "${DESCRIPTION}" | sed -e "s:-g${VCS_SHORT_HASH}\$::" -e 's:-[0-9]*$::')"
-
- # Distance to last tag or an alias of VCS_NUM if there is no tag
- if [ ! -z "${DESCRIPTION}" ]; then
- VCS_TICK="$(echo "${DESCRIPTION}" | sed -e "s:${VCS_TAG}-::" -e "s:-g${VCS_SHORT_HASH}::")"
- else
- VCS_TICK="${VCS_NUM}"
- fi
-
- # Date of the current commit
- VCS_DATE="$(TZ=UTC git show -s --date=iso-strict-local --pretty=format:%ad | sed -e 's|+00:00|Z|')"
- if [ -z "${VCS_DATE}" ]; then
- echo "warning: Action stamps require git version 2.7+." 1>&2
- VCS_DATE="$(git log -1 --pretty=format:%ci | sed -e 's: :T:' -e 's: ::' -e 's|+00:00|Z|')"
- local ASdis="1"
- fi
-
- # Action Stamp
- if [ -z "${ASdis}" ]; then
- VCS_ACTION_STAMP="${VCS_DATE}!$(git show -s --pretty=format:%cE)"
- else
- VCS_ACTION_STAMP=""
- fi
-
- cd "${oldPath}"
-}
-
-# For hg repos
-# shellcheck disable=SC2039,SC2164
-hgRepo() {
- local oldPath="${PWD}"
-
- cd "$(hg root)"
-
- VCS_TYPE="hg"
-
- VCS_BASENAME="$(basename "${PWD}")"
-
- VCS_UUID="$(hg log -r "0" -l 1 --template '{node}\n')"
-
- # Is the working copy clean?
- test -z "$(hg status -duram)"
- VCS_WC_MODIFIED="${?}"
-
- # Enumeration of changesets
- VCS_NUM="$(hg id -n | tr -d '+')"
-
- # The full revision hash
- VCS_FULL_HASH="$(hg log -r "${VCS_NUM}" -l 1 --template '{node}\n')"
-
- # The short hash
- VCS_SHORT_HASH="$(hg id -i | tr -d '+')"
-
- # Current bookmark (bookmarks are roughly equivalent to git's branches)
- # or branch if no bookmark
- VCS_BRANCH="$(hg id -B | cut -d ' ' -f 1)"
- # Fall back to the branch if there are no bookmarks
- if [ -z "${VCS_BRANCH}" ]; then
- VCS_BRANCH="$(hg id -b)"
- fi
-
- # Current or last tag ancestor (excluding auto tags, empty if no tags)
- VCS_TAG="$(hg log -r "${VCS_NUM}" -l 1 --template '{latesttag}\n' 2>/dev/null | sed -e 's:qtip::' -e 's:tip::' -e 's:qbase::' -e 's:qparent::' -e "s:$(hg --config 'extensions.color=' --config 'extensions.mq=' --color never qtop 2>/dev/null)::" | cut -d ' ' -f 1)"
-
- # Distance to last tag or an alias of VCS_NUM if there is no tag
- if [ ! -z "${VCS_TAG}" ]; then
- VCS_TICK="$(hg log -r "${VCS_NUM}" -l 1 --template '{latesttagdistance}\n' 2>/dev/null)"
- else
- VCS_TICK="${VCS_NUM}"
- fi
-
- # Date of the current commit
- VCS_DATE="$(hg log -r "${VCS_NUM}" -l 1 --template '{date|isodatesec}\n' 2>/dev/null | sed -e 's: :T:' -e 's: ::' -e 's|+00:00|Z|')"
-
- # Action Stamp
- VCS_ACTION_STAMP="$(TZ=UTC hg log -r "${VCS_NUM}" -l 1 --template '{date|localdate|rfc3339date}\n' 2>/dev/null | sed -e 's|+00:00|Z|')!$(hg log -r "${VCS_NUM}" -l 1 --template '{author|email}\n' 2>/dev/null)"
-
- cd "${oldPath}"
-}
-
-# For bzr repos
-# shellcheck disable=SC2039,SC2164
-bzrRepo() {
- local oldPath="${PWD}"
-
- cd "$(bzr root)"
-
- VCS_TYPE="bzr"
-
- VCS_BASENAME="$(basename "${PWD}")"
-
- # Currently unimplemented because more investigation is needed.
- VCS_UUID=""
-
- # Is the working copy clean?
- bzr version-info --custom --template='{clean}\n' | grep -q '1'
- VCS_WC_MODIFIED="${?}"
-
- # Enumeration of changesets
- VCS_NUM="$(bzr revno)"
-
- # The full revision hash
- VCS_FULL_HASH="$(bzr version-info --custom --template='{revision_id}\n')"
-
- # The short hash
- VCS_SHORT_HASH="${VCS_NUM}"
-
- # Nick of the current branch
- VCS_BRANCH="$(bzr nick)"
-
- # Current or last tag ancestor (excluding auto tags, empty if no tags)
- VCS_TAG="$(bzr tags --sort=time | sed '/?$/d' | tail -n1 | cut -d ' ' -f1)"
-
- # Distance to last tag or an alias of VCS_NUM if there is no tag
- if [ ! -z "${VCS_TAG}" ]; then
- VCS_TICK="$(bzr log --line -r "tag:${VCS_TAG}.." | tail -n +2 | wc -l | sed -e 's:^ *::')"
- else
- VCS_TICK="${VCS_NUM}"
- fi
-
- # Date of the current commit
- VCS_DATE="$(bzr version-info --custom --template='{date}\n' | sed -e 's: :T:' -e 's: ::')"
-
- # Action Stamp
- # Currently unimplemented because more investigation is needed.
- VCS_ACTION_STAMP=""
-
- cd "${oldPath}"
-}
-
-# For svn repos
-# shellcheck disable=SC2039,SC2164,SC2155
-svnRepo() {
- local oldPath="${PWD}"
-
- VCS_TYPE="svn"
-
- case "${PWD}" in
- /*trunk*|/*branches*|/*tags*)
- local fn="${PWD}"
- while [ "$(basename "${fn}")" != 'trunk' ] && [ "$(basename "${fn}")" != 'branches' ] && [ "$(basename "${fn}")" != 'tags' ] && [ "$(basename "${fn}")" != '/' ]; do
- local fn="$(dirname "${fn}")"
- done
- local fn="$(dirname "${fn}")"
- if [ "${fn}" = '/' ]; then
- VCS_BASENAME="$(basename "${PWD}")"
- else
- VCS_BASENAME="$(basename "${fn}")"
- fi
- ;;
- *) VCS_BASENAME="$(basename "${PWD}")" ;;
- esac
-
- VCS_UUID="$(svn info --xml | sed -n -e 's:::' -e 's:::p')"
-
- # Cache svnversion output
- local SVNVERSION="$(svnversion)"
-
- # Is the working copy clean?
- echo "${SVNVERSION}" | grep -q "M"
- case "${?}" in
- 0)
- VCS_WC_MODIFIED="1"
- ;;
- 1)
- if [ ! -z "${UNTRACKEDFILES}" ]; then
- # `svnversion` does not detect untracked files and `svn status` is really slow, so only run it if we really have to.
- if [ -z "$(svn status)" ]; then
- VCS_WC_MODIFIED="0"
- else
- VCS_WC_MODIFIED="1"
- fi
- else
- VCS_WC_MODIFIED="0"
- fi
- ;;
- esac
-
- # Enumeration of changesets
- VCS_NUM="$(echo "${SVNVERSION}" | cut -d : -f 1 | sed -e 's:M::' -e 's:S::' -e 's:P::')"
-
- # The full revision hash
- VCS_FULL_HASH="${SVNVERSION}"
-
- # The short hash
- VCS_SHORT_HASH="${VCS_NUM}"
-
- # Current branch
- case "${PWD}" in
- /*trunk*|/*branches*|/*tags*)
- local lastbase=""
- local fn="${PWD}"
- while :
- do
- base="$(basename "${fn}")"
- if [ "${base}" = 'trunk' ]; then
- VCS_BRANCH='trunk'
- break
- elif [ "${base}" = 'branches' ] || [ "${base}" = 'tags' ]; then
- VCS_BRANCH="${lastbase}"
- break
- elif [ "${base}" = '/' ]; then
- VCS_BRANCH=""
- break
- fi
- local lastbase="${base}"
- local fn="$(dirname "${fn}")"
- done
- ;;
- *) VCS_BRANCH="" ;;
- esac
-
- # Current or last tag ancestor (empty if no tags). But "current
- # tag" can't be extracted reliably because Subversion doesn't
- # have tags the way other VCSes do.
- VCS_TAG=""
- VCS_TICK=""
-
- # Date of the current commit
- VCS_DATE="$(svn info --xml | sed -n -e 's:::' -e 's:::p')"
-
- # Action Stamp
- VCS_ACTION_STAMP="${VCS_DATE}!$(svn log --xml -l 1 -r "${VCS_SHORT_HASH}" | sed -n -e 's:::' -e 's:::p')"
-
- cd "${oldPath}"
-}
-
-
-# Functions to output data in different formats.
-# For bash output
-shOutput() {
- cat > "${TARGETFILE}" << EOF
-# ${GENERATED_HEADER}
-
-VCS_TYPE="${VCS_TYPE}"
-VCS_BASENAME="${VCS_BASENAME}"
-VCS_UUID="${VCS_UUID}"
-VCS_NUM="${VCS_NUM}"
-VCS_DATE="${VCS_DATE}"
-VCS_BRANCH="${VCS_BRANCH}"
-VCS_TAG="${VCS_TAG}"
-VCS_TICK="${VCS_TICK}"
-VCS_EXTRA="${VCS_EXTRA}"
-
-VCS_ACTION_STAMP="${VCS_ACTION_STAMP}"
-VCS_FULL_HASH="${VCS_FULL_HASH}"
-VCS_SHORT_HASH="${VCS_SHORT_HASH}"
-
-VCS_WC_MODIFIED="${VCS_WC_MODIFIED}"
-
-# end
-EOF
-}
-
-# For source C output
-cOutput() {
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-
-const char *VCS_TYPE = "${VCS_TYPE}";
-const char *VCS_BASENAME = "${VCS_BASENAME}";
-const char *VCS_UUID = "${VCS_UUID}";
-const int VCS_NUM = ${VCS_NUM};
-const char *VCS_DATE = "${VCS_DATE}";
-const char *VCS_BRANCH = "${VCS_BRANCH}";
-const char *VCS_TAG = "${VCS_TAG}";
-const int VCS_TICK = ${VCS_TICK};
-const char *VCS_EXTRA = "${VCS_EXTRA}";
-
-const char *VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}";
-const char *VCS_FULL_HASH = "${VCS_FULL_HASH}";
-const char *VCS_SHORT_HASH = "${VCS_SHORT_HASH}";
-
-const int VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-
-/* end */
-EOF
-}
-
-# For header output
-hOutput() {
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-#ifndef AUTOREVISION_H
-#define AUTOREVISION_H
-
-#define VCS_TYPE "${VCS_TYPE}"
-#define VCS_BASENAME "${VCS_BASENAME}"
-#define VCS_UUID "${VCS_UUID}"
-#define VCS_NUM ${VCS_NUM}
-#define VCS_DATE "${VCS_DATE}"
-#define VCS_BRANCH "${VCS_BRANCH}"
-#define VCS_TAG "${VCS_TAG}"
-#define VCS_TICK ${VCS_TICK}
-#define VCS_EXTRA "${VCS_EXTRA}"
-
-#define VCS_ACTION_STAMP "${VCS_ACTION_STAMP}"
-#define VCS_FULL_HASH "${VCS_FULL_HASH}"
-#define VCS_SHORT_HASH "${VCS_SHORT_HASH}"
-
-#define VCS_WC_MODIFIED ${VCS_WC_MODIFIED}
-
-#endif
-
-/* end */
-EOF
-}
-
-# A header output for use with xcode to populate info.plist strings
-xcodeOutput() {
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-#ifndef AUTOREVISION_H
-#define AUTOREVISION_H
-
-#define VCS_TYPE ${VCS_TYPE}
-#define VCS_BASENAME ${VCS_BASENAME}
-#define VCS_UUID ${VCS_UUID}
-#define VCS_NUM ${VCS_NUM}
-#define VCS_DATE ${VCS_DATE}
-#define VCS_BRANCH ${VCS_BRANCH}
-#define VCS_TAG ${VCS_TAG}
-#define VCS_TICK ${VCS_TICK}
-#define VCS_EXTRA ${VCS_EXTRA}
-
-#define VCS_ACTION_STAMP ${VCS_ACTION_STAMP}
-#define VCS_FULL_HASH ${VCS_FULL_HASH}
-#define VCS_SHORT_HASH ${VCS_SHORT_HASH}
-
-#define VCS_WC_MODIFIED ${VCS_WC_MODIFIED}
-
-#endif
-
-/* end */
-EOF
-}
-
-# For Swift output
-swiftOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- # For values that may not exist depending on the type of repo we
- # have read from, set them to `nil` when they are empty.
- if [ -z "${VCS_UUID}" ]; then
- VCS_UUID="nil"
- else
- VCS_UUID="\"${VCS_UUID}\""
- fi
- if [ -z "${VCS_TAG}" ]; then
- VCS_TAG="nil"
- else
- VCS_TAG="\"${VCS_TAG}\""
- fi
- : "${VCS_TICK:="nil"}"
- if [ -z "${VCS_EXTRA}" ]; then
- VCS_EXTRA="nil"
- else
- VCS_EXTRA="\"${VCS_EXTRA}\""
- fi
- if [ -z "${VCS_ACTION_STAMP}" ]; then
- VCS_ACTION_STAMP="nil"
- else
- VCS_ACTION_STAMP="\"${VCS_ACTION_STAMP}\""
- fi
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-
-let VCS_TYPE = "${VCS_TYPE}"
-let VCS_BASENAME = "${VCS_BASENAME}"
-let VCS_UUID: String? = ${VCS_UUID}
-let VCS_NUM: Int = ${VCS_NUM}
-let VCS_DATE = "${VCS_DATE}"
-let VCS_BRANCH: String = "${VCS_BRANCH}"
-let VCS_TAG: String? = ${VCS_TAG}
-let VCS_TICK: Int? = ${VCS_TICK}
-let VCS_EXTRA: String? = ${VCS_EXTRA}
-
-let VCS_ACTION_STAMP: String? = ${VCS_ACTION_STAMP}
-let VCS_FULL_HASH: String = "${VCS_FULL_HASH}"
-let VCS_SHORT_HASH: String = "${VCS_SHORT_HASH}"
-
-let VCS_WC_MODIFIED: Bool = ${VCS_WC_MODIFIED}
-
-/* end */
-EOF
-}
-
-# For Python output
-pyOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="False" ;;
- 1) VCS_WC_MODIFIED="True" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-# ${GENERATED_HEADER}
-
-VCS_TYPE = "${VCS_TYPE}"
-VCS_BASENAME = "${VCS_BASENAME}"
-VCS_UUID = "${VCS_UUID}"
-VCS_NUM = ${VCS_NUM}
-VCS_DATE = "${VCS_DATE}"
-VCS_BRANCH = "${VCS_BRANCH}"
-VCS_TAG = "${VCS_TAG}"
-VCS_TICK = ${VCS_TICK}
-VCS_EXTRA = "${VCS_EXTRA}"
-
-VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"
-VCS_FULL_HASH = "${VCS_FULL_HASH}"
-VCS_SHORT_HASH = "${VCS_SHORT_HASH}"
-
-VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}
-
-# end
-EOF
-}
-
-# For Perl output
-plOutput() {
- cat << EOF
-# ${GENERATED_HEADER}
-
-\$VCS_TYPE = '${VCS_TYPE}';
-\$VCS_BASENAME = '${VCS_BASENAME}';
-\$VCS_UUID = '${VCS_UUID}';
-\$VCS_NUM = ${VCS_NUM};
-\$VCS_DATE = '${VCS_DATE}';
-\$VCS_BRANCH = '${VCS_BRANCH}';
-\$VCS_TAG = '${VCS_TAG}';
-\$VCS_TICK = ${VCS_TICK};
-\$VCS_EXTRA = '${VCS_EXTRA}';
-
-\$VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}';
-\$VCS_FULL_HASH = '${VCS_FULL_HASH}';
-\$VCS_SHORT_HASH = '${VCS_SHORT_HASH}';
-
-\$VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-
-# end
-1;
-EOF
-}
-
-# For lua output
-luaOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- cat > "${TARGETFILE}" << EOF
--- ${GENERATED_HEADER}
-
-VCS_TYPE = "${VCS_TYPE}"
-VCS_BASENAME = "${VCS_BASENAME}"
-VCS_UUID = "${VCS_UUID}"
-VCS_NUM = ${VCS_NUM}
-VCS_DATE = "${VCS_DATE}"
-VCS_BRANCH = "${VCS_BRANCH}"
-VCS_TAG = "${VCS_TAG}"
-VCS_TICK = ${VCS_TICK}
-VCS_EXTRA = "${VCS_EXTRA}"
-
-VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"
-VCS_FULL_HASH = "${VCS_FULL_HASH}"
-VCS_SHORT_HASH = "${VCS_SHORT_HASH}"
-
-VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}
-
--- end
-EOF
-}
-
-# For php output
-phpOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- cat > "${TARGETFILE}" << EOF
- "${VCS_TYPE}",
- "VCS_BASENAME" => "${VCS_BASENAME}",
- "VCS_UUID" => "${VCS_UUID}",
- "VCS_NUM" => ${VCS_NUM},
- "VCS_DATE" => "${VCS_DATE}",
- "VCS_BRANCH" => "${VCS_BRANCH}",
- "VCS_TAG" => "${VCS_TAG}",
- "VCS_TICK" => ${VCS_TICK},
- "VCS_EXTRA" => "${VCS_EXTRA}",
- "VCS_ACTION_STAMP" => "${VCS_ACTION_STAMP}",
- "VCS_FULL_HASH" => "${VCS_FULL_HASH}",
- "VCS_SHORT_HASH" => "${VCS_SHORT_HASH}",
- "VCS_WC_MODIFIED" => ${VCS_WC_MODIFIED}
-);
-
-# end
-?>
-EOF
-}
-
-# For ini output
-iniOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-; ${GENERATED_HEADER}
-[VCS]
-VCS_TYPE = "${VCS_TYPE}"
-VCS_BASENAME = "${VCS_BASENAME}"
-VCS_UUID = "${VCS_UUID}"
-VCS_NUM = ${VCS_NUM}
-VCS_DATE = "${VCS_DATE}"
-VCS_BRANCH = "${VCS_BRANCH}"
-VCS_TAG = "${VCS_TAG}"
-VCS_TICK = ${VCS_TICK}
-VCS_EXTRA = "${VCS_EXTRA}"
-VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"
-VCS_FULL_HASH = "${VCS_FULL_HASH}"
-VCS_SHORT_HASH = "${VCS_SHORT_HASH}"
-VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}
-; end
-EOF
-}
-
-# For javascript output
-jsOutput() {
- case "${VCS_WC_MODIFIED}" in
- 1) VCS_WC_MODIFIED="true" ;;
- 0) VCS_WC_MODIFIED="false" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-/** ${GENERATED_HEADER} */
-
-var autorevision = {
- VCS_TYPE: "${VCS_TYPE}",
- VCS_BASENAME: "${VCS_BASENAME}",
- VCS_UUID: "${VCS_UUID}",
- VCS_NUM: ${VCS_NUM},
- VCS_DATE: "${VCS_DATE}",
- VCS_BRANCH: "${VCS_BRANCH}",
- VCS_TAG: "${VCS_TAG}",
- VCS_TICK: ${VCS_TICK},
- VCS_EXTRA: "${VCS_EXTRA}",
-
- VCS_ACTION_STAMP: "${VCS_ACTION_STAMP}",
- VCS_FULL_HASH: "${VCS_FULL_HASH}",
- VCS_SHORT_HASH: "${VCS_SHORT_HASH}",
-
- VCS_WC_MODIFIED: ${VCS_WC_MODIFIED}
-};
-
-/** Node.js compatibility */
-if (typeof module !== 'undefined') {
- module.exports = autorevision;
-}
-
-/** end */
-EOF
-}
-
-# For JSON output
-jsonOutput() {
- case "${VCS_WC_MODIFIED}" in
- 1) VCS_WC_MODIFIED="true" ;;
- 0) VCS_WC_MODIFIED="false" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-{
- "_comment": "${GENERATED_HEADER}",
- "VCS_TYPE": "${VCS_TYPE}",
- "VCS_BASENAME": "${VCS_BASENAME}",
- "VCS_UUID": "${VCS_UUID}",
- "VCS_NUM": ${VCS_NUM},
- "VCS_DATE": "${VCS_DATE}",
- "VCS_BRANCH":"${VCS_BRANCH}",
- "VCS_TAG": "${VCS_TAG}",
- "VCS_TICK": ${VCS_TICK},
- "VCS_EXTRA": "${VCS_EXTRA}",
-
- "VCS_ACTION_STAMP": "${VCS_ACTION_STAMP}",
- "VCS_FULL_HASH": "${VCS_FULL_HASH}",
- "VCS_SHORT_HASH": "${VCS_SHORT_HASH}",
-
- "VCS_WC_MODIFIED": ${VCS_WC_MODIFIED}
-}
-EOF
-}
-
-# For Java output
-javaOutput() {
- case "${VCS_WC_MODIFIED}" in
- 1) VCS_WC_MODIFIED="true" ;;
- 0) VCS_WC_MODIFIED="false" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-
-public class autorevision {
- public static final String VCS_TYPE = "${VCS_TYPE}";
- public static final String VCS_BASENAME = "${VCS_BASENAME}";
- public static final String VCS_UUID = "${VCS_UUID}";
- public static final long VCS_NUM = ${VCS_NUM};
- public static final String VCS_DATE = "${VCS_DATE}";
- public static final String VCS_BRANCH = "${VCS_BRANCH}";
- public static final String VCS_TAG = "${VCS_TAG}";
- public static final long VCS_TICK = ${VCS_TICK};
- public static final String VCS_EXTRA = "${VCS_EXTRA}";
-
- public static final String VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}";
- public static final String VCS_FULL_HASH = "${VCS_FULL_HASH}";
- public static final String VCS_SHORT_HASH = "${VCS_SHORT_HASH}";
-
- public static final boolean VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-}
-EOF
-}
-
-# For Java properties output
-javapropOutput() {
- case "${VCS_WC_MODIFIED}" in
- 1) VCS_WC_MODIFIED="true" ;;
- 0) VCS_WC_MODIFIED="false" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-# ${GENERATED_HEADER}
-
-VCS_TYPE=${VCS_TYPE}
-VCS_BASENAME=${VCS_BASENAME}
-VCS_UUID=${VCS_UUID}
-VCS_NUM=${VCS_NUM}
-VCS_DATE=${VCS_DATE}
-VCS_BRANCH=${VCS_BRANCH}
-VCS_TAG=${VCS_TAG}
-VCS_TICK=${VCS_TICK}
-VCS_EXTRA=${VCS_EXTRA}
-
-VCS_ACTION_STAMP=${VCS_ACTION_STAMP}
-VCS_FULL_HASH=${VCS_FULL_HASH}
-VCS_SHORT_HASH=${VCS_SHORT_HASH}
-
-VCS_WC_MODIFIED=${VCS_WC_MODIFIED}
-EOF
-}
-
-# For m4 output
-m4Output() {
- cat > "${TARGETFILE}" << EOF
-dnl ${GENERATED_HEADER}
-define(\`VCS_TYPE', \`${VCS_TYPE}')dnl
-define(\`VCS_BASENAME', \`${VCS_BASENAME}')dnl
-define(\`VCS_UUID', \`${VCS_UUID}')dnl
-define(\`VCS_NUM', \`${VCS_NUM}')dnl
-define(\`VCS_DATE', \`${VCS_DATE}')dnl
-define(\`VCS_BRANCH', \`${VCS_BRANCH}')dnl
-define(\`VCS_TAG', \`${VCS_TAG}')dnl
-define(\`VCS_TICK', \`${VCS_TICK}')dnl
-define(\`VCS_EXTRA', \`${VCS_EXTRA}')dnl
-define(\`VCS_ACTIONSTAMP', \`${VCS_ACTION_STAMP}')dnl
-define(\`VCS_FULLHASH', \`${VCS_FULL_HASH}')dnl
-define(\`VCS_SHORTHASH', \`${VCS_SHORT_HASH}')dnl
-define(\`VCS_WC_MODIFIED', \`${VCS_WC_MODIFIED}')dnl
-EOF
-}
-
-# For (La)TeX output
-texOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-% ${GENERATED_HEADER}
-\def \vcsType {${VCS_TYPE}}
-\def \vcsBasename {${VCS_BASENAME}}
-\def \vcsUUID {${VCS_UUID}}
-\def \vcsNum {${VCS_NUM}}
-\def \vcsDate {${VCS_DATE}}
-\def \vcsBranch {${VCS_BRANCH}}
-\def \vcsTag {${VCS_TAG}}
-\def \vcsTick {${VCS_TICK}}
-\def \vcsExtra {${VCS_EXTRA}}
-\def \vcsACTIONSTAMP {${VCS_ACTION_STAMP}}
-\def \vcsFullHash {${VCS_FULL_HASH}}
-\def \vcsShortHash {${VCS_SHORT_HASH}}
-\def \vcsWCModified {${VCS_WC_MODIFIED}}
-\endinput
-EOF
-}
-
-# For scheme output
-schemeOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="#f" ;;
- 1) VCS_WC_MODIFIED="#t" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-;; ${GENERATED_HEADER}
-(define VCS_TYPE "${VCS_TYPE}")
-(define VCS_BASENAME "${VCS_BASENAME}")
-(define VCS_UUID "${VCS_UUID}")
-(define VCS_NUM ${VCS_NUM})
-(define VCS_DATE "${VCS_DATE}")
-(define VCS_BRANCH "${VCS_BRANCH}")
-(define VCS_TAG "${VCS_TAG}")
-(define VCS_TICK ${VCS_TICK})
-(define VCS_EXTRA "${VCS_EXTRA}")
-
-(define VCS_ACTION_STAMP "${VCS_ACTION_STAMP}")
-(define VCS_FULL_HASH "${VCS_FULL_HASH}")
-(define VCS_SHORT_HASH "${VCS_SHORT_HASH}")
-
-(define VCS_WC_MODIFIED ${VCS_WC_MODIFIED})
-;; end
-EOF
-}
-
-# For clojure output
-clojureOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="false" ;;
- 1) VCS_WC_MODIFIED="true" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-;; ${GENERATED_HEADER}
-(def VCS_TYPE "${VCS_TYPE}")
-(def VCS_BASENAME "${VCS_BASENAME}")
-(def VCS_UUID "${VCS_UUID}")
-(def VCS_NUM ${VCS_NUM})
-(def VCS_DATE "${VCS_DATE}")
-(def VCS_BRANCH "${VCS_BRANCH}")
-(def VCS_TAG "${VCS_TAG}")
-(def VCS_TICK ${VCS_TICK})
-(def VCS_EXTRA "${VCS_EXTRA}")
-
-(def VCS_ACTION_STAMP "${VCS_ACTION_STAMP}")
-(def VCS_FULL_HASH "${VCS_FULL_HASH}")
-(def VCS_SHORT_HASH "${VCS_SHORT_HASH}")
-
-(def VCS_WC_MODIFIED ${VCS_WC_MODIFIED})
-;; end
-EOF
-}
-
-# For rpm spec file output
-rpmOutput() {
- cat > "${TARGETFILE}" << EOF
-# ${GENERATED_HEADER}
-$([ "${VCS_TYPE}" ] && echo "%define vcs_type ${VCS_TYPE}")
-$([ "${VCS_BASENAME}" ] && echo "%define vcs_basename ${VCS_BASENAME}")
-$([ "${VCS_UUID}" ] && echo "%define vcs_uuid ${VCS_UUID}")
-$([ "${VCS_NUM}" ] && echo "%define vcs_num ${VCS_NUM}")
-$([ "${VCS_DATE}" ] && echo "%define vcs_date ${VCS_DATE}")
-$([ "${VCS_BRANCH}" ] && echo "%define vcs_branch ${VCS_BRANCH}")
-$([ "${VCS_TAG}" ] && echo "%define vcs_tag ${VCS_TAG}")
-$([ "${VCS_TICK}" ] && echo "%define vcs_tick ${VCS_TICK}")
-$([ "${VCS_EXTRA}" ] && echo "%define vcs_extra ${VCS_EXTRA}")
-
-$([ "${VCS_ACTION_STAMP}" ] && echo "%define vcs_action_stamp ${VCS_ACTION_STAMP}")
-$([ "${VCS_FULL_HASH}" ] && echo "%define vcs_full_hash ${VCS_FULL_HASH}")
-$([ "${VCS_SHORT_HASH}" ] && echo "%define vcs_short_hash ${VCS_SHORT_HASH}")
-
-$([ "${VCS_WC_MODIFIED}" ] && echo "%define vcs_wc_modified ${VCS_WC_MODIFIED}")
-# end
-EOF
-}
-
-# shellcheck disable=SC2155,SC2039
-hppOutput() {
- local NAMESPACE="$(echo "${VCS_BASENAME}" | sed -e 's:_::g' | tr '[:lower:]' '[:upper:]')"
- cat > "${TARGETFILE}" << EOF
-/* ${GENERATED_HEADER} */
-
-#ifndef ${NAMESPACE}_AUTOREVISION_H
-#define ${NAMESPACE}_AUTOREVISION_H
-
-#include
-
-namespace $(echo "${NAMESPACE}" | tr '[:upper:]' '[:lower:]')
-{
- const std::string VCS_TYPE = "${VCS_TYPE}";
- const std::string VCS_BASENAME = "${VCS_BASENAME}";
- const std::string VCS_UUID = "${VCS_UUID}";
- const int VCS_NUM = ${VCS_NUM};
- const std::string VCS_DATE = "${VCS_DATE}";
- const std::string VCS_BRANCH = "${VCS_BRANCH}";
- const std::string VCS_TAG = "${VCS_TAG}";
- const int VCS_TICK = ${VCS_TICK};
- const std::string VCS_EXTRA = "${VCS_EXTRA}";
-
- const std::string VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}";
- const std::string VCS_FULL_HASH = "${VCS_FULL_HASH}";
- const std::string VCS_SHORT_HASH = "${VCS_SHORT_HASH}";
-
- const int VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-}
-
-#endif
-
-/* end */
-EOF
-}
-
-matlabOutput() {
- case "${VCS_WC_MODIFIED}" in
- 0) VCS_WC_MODIFIED="FALSE" ;;
- 1) VCS_WC_MODIFIED="TRUE" ;;
- esac
- cat > "${TARGETFILE}" << EOF
-% ${GENERATED_HEADER}
-
-VCS_TYPE = '${VCS_TYPE}';
-VCS_BASENAME = '${VCS_BASENAME}';
-VCS_UUID = '${VCS_UUID}';
-VCS_NUM = ${VCS_NUM};
-VCS_DATE = '${VCS_DATE}';
-VCS_BRANCH = '${VCS_BRANCH}';
-VCS_TAG = '${VCS_TAG}';
-VCS_TICK = ${VCS_TICK};
-VCS_EXTRA = '${VCS_EXTRA}';
-
-VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}';
-VCS_FULL_HASH = '${VCS_FULL_HASH}';
-VCS_SHORT_HASH = '${VCS_SHORT_HASH}';
-
-VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-
-% end
-EOF
-}
-
-octaveOutput() {
- cat > "${TARGETFILE}" << EOF
-% ${GENERATED_HEADER}
-
-VCS_TYPE = '${VCS_TYPE}';
-VCS_BASENAME = '${VCS_BASENAME}';
-VCS_UUID = '${VCS_UUID}';
-VCS_NUM = ${VCS_NUM};
-VCS_DATE = '${VCS_DATE}';
-VCS_BRANCH = '${VCS_BRANCH}';
-VCS_TAG = '${VCS_TAG}';
-VCS_TICK = ${VCS_TICK};
-VCS_EXTRA = '${VCS_EXTRA}';
-
-VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}';
-VCS_FULL_HASH = '${VCS_FULL_HASH}';
-VCS_SHORT_HASH = '${VCS_SHORT_HASH}';
-
-VCS_WC_MODIFIED = ${VCS_WC_MODIFIED};
-
-% end
-EOF
-}
-
-cmakeOutput() {
- cat > "${TARGETFILE}" << EOF
-# ${GENERATED_HEADER}
-
-set(VCS_TYPE ${VCS_TYPE})
-set(VCS_BASENAME ${VCS_BASENAME})
-set(VCS_UUID ${VCS_UUID})
-set(VCS_NUM ${VCS_NUM})
-set(VCS_DATE ${VCS_DATE})
-set(VCS_BRANCH ${VCS_BRANCH})
-set(VCS_TAG ${VCS_TAG})
-set(VCS_TICK ${VCS_TICK})
-set(VCS_EXTRA ${VCS_EXTRA})
-
-set(VCS_ACTION_STAMP ${VCS_ACTION_STAMP})
-set(VCS_FULL_HASH ${VCS_FULL_HASH})
-set(VCS_SHORT_HASH ${VCS_SHORT_HASH})
-
-set(VCS_WC_MODIFIED ${VCS_WC_MODIFIED})
-
-# end
-EOF
-}
-
-
-# Helper functions
-# Count path segments
-# shellcheck disable=SC2039
-pathSegment() {
- local pathz="${1}"
- local depth="0"
-
- if [ ! -z "${pathz}" ]; then
- # Continue until we are at / or there are no path separators left.
- while [ ! "${pathz}" = "/" ] && [ ! "${pathz}" = "$(echo "${pathz}" | sed -e 's:/::')" ]; do
- pathz="$(dirname "${pathz}")"
- depth="$((depth+1))"
- done
- fi
- echo "${depth}"
-}
-
-# Largest of four numbers
-# shellcheck disable=SC2039
-multiCompare() {
- local larger="${1}"
- local numA="${2}"
- local numB="${3}"
- local numC="${4}"
-
- [ "${numA}" -gt "${larger}" ] && larger="${numA}"
- [ "${numB}" -gt "${larger}" ] && larger="${numB}"
- [ "${numC}" -gt "${larger}" ] && larger="${numC}"
- echo "${larger}"
-}
-
-# Test for repositories
-# shellcheck disable=SC2155,SC2039
-repoTest() {
- REPONUM="0"
- if [ ! -z "$(git rev-parse HEAD 2>/dev/null)" ]; then
- local gitPath="$(git rev-parse --show-toplevel)"
- local gitDepth="$(pathSegment "${gitPath}")"
- REPONUM="$((REPONUM+1))"
- else
- local gitDepth="0"
- fi
- if [ ! -z "$(hg root 2>/dev/null)" ]; then
- local hgPath="$(hg root 2>/dev/null)"
- local hgDepth="$(pathSegment "${hgPath}")"
- REPONUM="$((REPONUM+1))"
- else
- local hgDepth="0"
- fi
- if [ ! -z "$(bzr root 2>/dev/null)" ]; then
- local bzrPath="$(bzr root 2>/dev/null)"
- local bzrDepth="$(pathSegment "${bzrPath}")"
- REPONUM="$((REPONUM+1))"
- else
- local bzrDepth="0"
- fi
- if [ ! -z "$(svn info 2>/dev/null)" ]; then
- local stringz=""
- local stringx=""
- local svnPath="$(svn info --xml | sed -n -e "s:${stringz}::" -e "s:${stringx}::p")"
- # An old enough svn will not be able give us a path; default
- # to 1 for that case.
- if [ -z "${svnPath}" ]; then
- local svnDepth="1"
- else
- local svnDepth="$(pathSegment "${svnPath}")"
- fi
- REPONUM="$((REPONUM+1))"
- else
- local svnDepth="0"
- fi
-
- # Do not do more work then we have to.
- if [ "${REPONUM}" = "0" ]; then
- return
- fi
-
- # Figure out which repo is the deepest and use it.
- local wonRepo="$(multiCompare "${gitDepth}" "${hgDepth}" "${bzrDepth}" "${svnDepth}")"
- if [ "${wonRepo}" = "${gitDepth}" ]; then
- gitRepo
- elif [ "${wonRepo}" = "${hgDepth}" ]; then
- hgRepo
- elif [ "${wonRepo}" = "${bzrDepth}" ]; then
- bzrRepo
- elif [ "${wonRepo}" = "${svnDepth}" ]; then
- svnRepo
- fi
-}
-
-
-
-# Detect which repos we are in and gather data.
-# shellcheck source=/dev/null
-if [ -f "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then
- # When requested only read from the cache to populate our symbols.
- . "${CACHEFILE}"
-else
- # If a value is not set through the environment set VCS_EXTRA to nothing.
- : "${VCS_EXTRA:=""}"
- repoTest
-
- if [ -f "${CACHEFILE}" ] && [ "${REPONUM}" = "0" ]; then
- # We are not in a repo; try to use a previously generated cache to populate our symbols.
- . "${CACHEFILE}"
- # Do not overwrite the cache if we know we are not going to write anything new.
- CACHEFORCE="1"
- elif [ "${REPONUM}" = "0" ]; then
- echo "error: No repo or cache detected." 1>&2
- exit 1
- fi
-fi
-
-
-# -s output is handled here.
-if [ ! -z "${VAROUT}" ]; then
- if [ "${VAROUT}" = "VCS_TYPE" ]; then
- echo "${VCS_TYPE}"
- elif [ "${VAROUT}" = "VCS_BASENAME" ]; then
- echo "${VCS_BASENAME}"
- elif [ "${VAROUT}" = "VCS_NUM" ]; then
- echo "${VCS_NUM}"
- elif [ "${VAROUT}" = "VCS_DATE" ]; then
- echo "${VCS_DATE}"
- elif [ "${VAROUT}" = "VCS_BRANCH" ]; then
- echo "${VCS_BRANCH}"
- elif [ "${VAROUT}" = "VCS_TAG" ]; then
- echo "${VCS_TAG}"
- elif [ "${VAROUT}" = "VCS_TICK" ]; then
- echo "${VCS_TICK}"
- elif [ "${VAROUT}" = "VCS_FULL_HASH" ]; then
- echo "${VCS_FULL_HASH}"
- elif [ "${VAROUT}" = "VCS_SHORT_HASH" ]; then
- echo "${VCS_SHORT_HASH}"
- elif [ "${VAROUT}" = "VCS_WC_MODIFIED" ]; then
- echo "${VCS_WC_MODIFIED}"
- elif [ "${VAROUT}" = "VCS_ACTION_STAMP" ]; then
- echo "${VCS_ACTION_STAMP}"
- else
- echo "error: Not a valid output symbol." 1>&2
- exit 1
- fi
-fi
-
-
-# Detect requested output type and use it.
-if [ ! -z "${AFILETYPE}" ]; then
- if [ "${AFILETYPE}" = "c" ]; then
- cOutput
- elif [ "${AFILETYPE}" = "h" ]; then
- hOutput
- elif [ "${AFILETYPE}" = "xcode" ]; then
- xcodeOutput
- elif [ "${AFILETYPE}" = "swift" ]; then
- swiftOutput
- elif [ "${AFILETYPE}" = "sh" ]; then
- shOutput
- elif [ "${AFILETYPE}" = "py" ] || [ "${AFILETYPE}" = "python" ]; then
- pyOutput
- elif [ "${AFILETYPE}" = "pl" ] || [ "${AFILETYPE}" = "perl" ]; then
- plOutput
- elif [ "${AFILETYPE}" = "lua" ]; then
- luaOutput
- elif [ "${AFILETYPE}" = "php" ]; then
- phpOutput
- elif [ "${AFILETYPE}" = "ini" ]; then
- iniOutput
- elif [ "${AFILETYPE}" = "js" ]; then
- jsOutput
- elif [ "${AFILETYPE}" = "json" ]; then
- jsonOutput
- elif [ "${AFILETYPE}" = "java" ]; then
- javaOutput
- elif [ "${AFILETYPE}" = "javaprop" ]; then
- javapropOutput
- elif [ "${AFILETYPE}" = "tex" ]; then
- texOutput
- elif [ "${AFILETYPE}" = "m4" ]; then
- m4Output
- elif [ "${AFILETYPE}" = "scheme" ]; then
- schemeOutput
- elif [ "${AFILETYPE}" = "clojure" ]; then
- clojureOutput
- elif [ "${AFILETYPE}" = "rpm" ]; then
- rpmOutput
- elif [ "${AFILETYPE}" = "hpp" ]; then
- hppOutput
- elif [ "${AFILETYPE}" = "matlab" ]; then
- matlabOutput
- elif [ "${AFILETYPE}" = "octave" ]; then
- octaveOutput
- elif [ "${AFILETYPE}" = "cmake" ]; then
- cmakeOutput
- else
- echo "error: Not a valid output type." 1>&2
- exit 1
- fi
-fi
-
-
-# If requested, make a cache file.
-if [ ! -z "${CACHEFILE}" ] && [ ! "${CACHEFORCE}" = "1" ]; then
- TARGETFILE="${CACHEFILE}.tmp"
- shOutput
-
- # Check to see if there have been any actual changes.
- if [ ! -f "${CACHEFILE}" ]; then
- mv -f "${CACHEFILE}.tmp" "${CACHEFILE}"
- elif cmp -s "${CACHEFILE}.tmp" "${CACHEFILE}"; then
- rm -f "${CACHEFILE}.tmp"
- else
- mv -f "${CACHEFILE}.tmp" "${CACHEFILE}"
- fi
-fi
diff --git a/ci/get-nprocessors.sh b/ci/get-nprocessors.sh
deleted file mode 100644
index 43635e7..0000000
--- a/ci/get-nprocessors.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 Google Inc.
-# All Rights Reserved.
-#
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# This file is typically sourced by another script.
-# if possible, ask for the precise number of processors,
-# otherwise take 2 processors as reasonable default; see
-# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization
-if [ -x /usr/bin/getconf ]; then
- NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN)
-else
- NPROCESSORS=2
-fi
-
-# as of 2017-09-04 Travis CI reports 32 processors, but GCC build
-# crashes if parallelized too much (maybe memory consumption problem),
-# so limit to 4 processors for the time being.
-if [ $NPROCESSORS -gt 4 ] ; then
- echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4."
- NPROCESSORS=4
-fi
diff --git a/ci/perpare_pulp3_netrc.sh b/ci/perpare_pulp3_netrc.sh
deleted file mode 100644
index 8414bbb..0000000
--- a/ci/perpare_pulp3_netrc.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env sh
-set -evx
-echo "machine ${PULP3_SERVER_URL}\nlogin ${PULP3_SERVER_LOGIN}\npassword ${PULP3_SERVER_PASSWORD}\n" > ~/.netrc
diff --git a/ci/travis.sh b/ci/travis.sh
deleted file mode 100644
index 6e9056b..0000000
--- a/ci/travis.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env sh
-set -evx
-
-chmod +x ci/get-nprocessors.sh
-. ci/get-nprocessors.sh
-
-# if possible, ask for the precise number of processors,
-# otherwise take 2 processors as reasonable default; see
-# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization
-if [ -x /usr/bin/getconf ]; then
- NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN)
-else
- NPROCESSORS=2
-fi
-
-# as of 2017-09-04 Travis CI reports 32 processors, but GCC build
-# crashes if parallelized too much (maybe memory consumption problem),
-# so limit to 4 processors for the time being.
-if [ $NPROCESSORS -gt 4 ] ; then
- echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4."
- NPROCESSORS=4
-fi
-
-# Tell make to use the processors. No preceding '-' required.
-MAKEFLAGS="j${NPROCESSORS}"
-export MAKEFLAGS
-
-env | sort
-
-# Set default values to OFF for these variables if not specified.
-: "${NO_EXCEPTION:=OFF}"
-: "${NO_RTTI:=OFF}"
-: "${COMPILER_IS_GNUCXX:=OFF}"
-
-# Install dependency from YUM
-if [ -n "${INSTALL_DEPENDENCY_LIBRARY}" ]; then
- yum install -y $INSTALL_DEPENDENCY_LIBRARY
- source /etc/profile.d/framework.sh
-fi
-
-if [ $ASAN_OPTION ] && [ -f "/opt/rh/devtoolset-7/enable" ] ;then
- source /opt/rh/devtoolset-7/enable
-fi
-
-mkdir build || true
-cd build
-
-cmake3 -DCMAKE_CXX_FLAGS=$CXX_FLAGS \
- -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
- -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
- -DASAN_OPTION=$ASAN_OPTION \
- -DVERSION_DAILY_BUILD=$TESTING_VERSION_BUILD \
- ..
-
-make
-
-if [ -n "${PACKAGE}" ]; then
- make package
-fi
-
-
-if [ -n "${UPLOAD}" ]; then
- cp ~/rpm_upload_tools.py ./
- python3 rpm_upload_tools.py ${PULP3_REPO_NAME} ${PULP3_DIST_NAME} *.rpm
-fi
-
-if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then
- rpm -i $SYMBOL_TARGET*debuginfo*.rpm
- _symbol_file=`find /usr/lib/debug/ -name "$SYMBOL_TARGET*.so*.debug"`
- cp $_symbol_file ${_symbol_file}info.${CI_COMMIT_SHORT_SHA}
- sentry-cli upload-dif -t elf ${_symbol_file}info.${CI_COMMIT_SHORT_SHA}
-fi
diff --git a/cmake/Package.cmake b/cmake/Package.cmake
deleted file mode 100644
index e112c59..0000000
--- a/cmake/Package.cmake
+++ /dev/null
@@ -1,57 +0,0 @@
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
- set(MY_RPM_NAME_PREFIX "lib${lib_name}-debug")
-else()
- set(MY_RPM_NAME_PREFIX "lib${lib_name}")
-endif()
-
-message(STATUS "Package: ${MY_RPM_NAME_PREFIX}")
-
-set(CPACK_PACKAGE_VECDOR "MESA")
-set(CPACK_PACKAGE_VERSION_MAJOR "${VERSION_MAJOR}")
-set(CPACK_PACKAGE_VERSION_MINOR "${VERSION_MINOR}")
-set(CPACK_PACKAGE_VERSION_PATCH "${VERSION_PATCH}.${VERSION_BUILD}")
-set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
-set(CPACK_PACKAGE_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_BUILD}")
-
-execute_process(COMMAND sh changelog.sh ${CMAKE_BINARY_DIR} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/cmake)
-SET(CPACK_RPM_CHANGELOG_FILE ${CMAKE_BINARY_DIR}/changelog.txt)
-
-# RPM Build
-set(CPACK_GENERATOR "RPM")
-set(CPACK_RPM_AUTO_GENERATED_FILE_NAME ON)
-set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
-set(CPACK_RPM_PACKAGE_VENDOR "MESA")
-set(CPACK_RPM_PACKAGE_AUTOREQPROV "yes")
-set(CPACK_RPM_PACKAGE_RELEASE_DIST "on")
-set(CPACK_RPM_DEBUGINFO_PACKAGE "on")
-
-set(CPACK_RPM_COMPONENT_INSTALL ON)
-set(CPACK_COMPONENTS_IGNORE_GROUPS 1)
-set(CPACK_COMPONENTS_GROUPING ONE_PER_GROUP)
-set(CPACK_COMPONENT_HEADER_DISPLAY_NAME "develop")
-
-set(CPACK_COMPONENT_LIBRARIES_REQUIRED TRUE)
-set(CPACK_RPM_LIBRARIES_PACKAGE_NAME ${MY_RPM_NAME_PREFIX})
-set(CPACK_COMPONENT_LIBRARIES_GROUP "LIBRARIES")
-
-set(CPACK_COMPONENT_HEADER_REQUIRED TRUE)
-set(CPACK_RPM_HEADER_PACKAGE_NAME "${MY_RPM_NAME_PREFIX}-devel")
-set(CPACK_COMPONENT_HEADER_GROUP "HEADER")
-
-set(CPACK_COMPONENT_TOOLS_REQUIRED TRUE)
-set(CPACK_RPM_TOOLS_PACKAGE_NAME "${MY_RPM_NAME_PREFIX}-tools")
-set(CPACK_COMPONENT_TOOLS_GROUP "TOOLS")
-
-set(CPACK_RPM_HEADER_PACKAGE_REQUIRES_PRE ${CPACK_RPM_LIBRARIES_PACKAGE_NAME})
-set(CPACK_RPM_HEADER_PACKAGE_CONFLICTS ${CPACK_RPM_HEADER_PACKAGE_NAME})
-
-set(CPACK_COMPONENTS_ALL LIBRARIES HEADER TOOLS)
-
-
-set(CPACK_BUILD_SOURCE_DIRS "${CMAKE_SOURCE_DIR}")
-
-# Must uninstall the debug package before install release package
-set(CPACK_RPM_PACKAGE_CONFLICTS ${MY_RPM_NAME_PREFIX})
-
-# set(CPACK_STRIP_FILES TRUE)
-include(CPack)
diff --git a/cmake/Version.cmake b/cmake/Version.cmake
deleted file mode 100644
index a47944c..0000000
--- a/cmake/Version.cmake
+++ /dev/null
@@ -1,54 +0,0 @@
-
-# Using autorevision.sh to generate version information
-
-set(__SOURCE_AUTORESIVISION ${CMAKE_SOURCE_DIR}/autorevision.sh)
-set(__AUTORESIVISION ${CMAKE_BINARY_DIR}/autorevision.sh)
-set(__VERSION_CACHE ${CMAKE_BINARY_DIR}/version.txt)
-set(__VERSION_CONFIG ${CMAKE_BINARY_DIR}/version.cmake)
-
-file(COPY ${__SOURCE_AUTORESIVISION} DESTINATION ${CMAKE_BINARY_DIR}
- FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE
- WORLD_READ WORLD_EXECUTE)
-
-# execute autorevision.sh to generate version information
-execute_process(COMMAND ${__AUTORESIVISION} -t cmake -o ${__VERSION_CACHE}
- OUTPUT_FILE ${__VERSION_CONFIG} ERROR_QUIET)
-include(${__VERSION_CONFIG})
-
-# extract major, minor, patch version from git tag
-string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" VERSION_MAJOR "${VCS_TAG}")
-string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" VERSION_MINOR "${VCS_TAG}")
-string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VCS_TAG}")
-string(REGEX REPLACE "[T\\:\\+\\-]" "" VERSION_DATE "${VCS_DATE}")
-
-if(VERSION_DAILY_BUILD)
- set(VERSION_PATCH ${VERSION_PATCH}.${VERSION_DATE})
-endif()
-
-if(NOT VERSION_MAJOR)
- set(VERSION_MAJOR 1)
-endif()
-
-if(NOT VERSION_MINOR)
- set(VERSION_MINOR 0)
-endif()
-
-if(NOT VERSION_PATCH)
- set(VERSION_PATCH 0)
-endif()
-
-set(VERSION "${VERSION_MAJOR}_${VERSION_MINOR}_${VERSION_PATCH}")
-set(VERSION_BUILD "${VCS_SHORT_HASH}")
-
-# print information
-message(STATUS "Version: ${VERSION}-${VERSION_BUILD}")
-
-option(DEFINE_GIT_VERSION "Set DEFINE_GIT_VERSION to TRUE or FALSE" TRUE)
-
-if(DEFINE_GIT_VERSION)
- set(GIT_VERSION
- "${VERSION}-${CMAKE_BUILD_TYPE}-${VERSION_BUILD}-${VCS_BRANCH}-${VCS_TAG}-${VCS_DATE}")
- string(REGEX REPLACE "[-:+/\\.]" "_" GIT_VERSION ${GIT_VERSION})
-
- add_definitions(-DGIT_VERSION=${GIT_VERSION})
-endif()
diff --git a/cmake/changelog.sh b/cmake/changelog.sh
deleted file mode 100644
index 9e5a277..0000000
--- a/cmake/changelog.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-work_path=$1
-branch=`git status | grep branch | awk '{print $NF}'`
-git log --branches=$branch --no-merges --date=local --show-signature --pretty="** %cd %an %ae %nhash: %H%ncommit:%n%B" | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/ -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' | sed 's/\*/ -/g' | sed 's/ - -/\*/g' > $work_path/changelog.txt
-
-#git log --branches=$branch --no-merges --date=local --show-signature --pretty="** %cd %an %ae %nhash: %H%ncommit:%n%B" | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/ -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' > $work_path/changelog.txt
diff --git a/inc/Maat_command.h b/inc/Maat_command.h
deleted file mode 100644
index 3b783a4..0000000
--- a/inc/Maat_command.h
+++ /dev/null
@@ -1,221 +0,0 @@
-#ifndef H_MAAT_COMMAND_H_INCLUDE
-#define H_MAAT_COMMAND_H_INCLUDE
-#ifdef __cplusplus
-extern "C"{
-#endif
-#include "Maat_rule.h"
-enum MAAT_OPERATION
-{
- MAAT_OP_DEL=0,
- MAAT_OP_ADD,
- MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
-};
-enum MAAT_GROUP_RELATION
-{
- PARENT_TYPE_COMPILE=0,
- PARENT_TYPE_GROUP
-};
-enum MAAT_REGION_TYPE
-{
- REGION_EXPR,
- REGION_IP,
- REGION_IP_PLUS,
- REGION_INTERVAL,
- REGION_DIGEST,
- REGION_SIMILARITY
-};
-enum MAAT_EXPR_TYPE
-{
- EXPR_TYPE_STRING=0,
- EXPR_TYPE_AND,
- EXPR_TYPE_REGEX,
- EXPR_TYPE_OFFSET
-};
-enum MAAT_MATCH_METHOD
-{
- MATCH_METHOD_SUB=0,
- MATCH_METHOD_SUFFIX,
- MATCH_METHOD_PREFIX,
- MATCH_METHOD_COMPLETE
-};
-
-enum MAAT_CASE_TYPE
-{
- UNCASE_PLAIN=0,
- CASE_HEXBIN,
- CASE_PLAIN
-};
-enum MAAT_ADDR_TYPE
-{
- ADDR_TYPE_IPv4=4,
- ADDR_TYPE_IPv6=6
-};
-enum MAAT_ADDR_DIRECTION
-{
- ADDR_DIR_DOUBLE=0,
- ADDR_DIR_SINGLE=1
-};
-struct Maat_rgn_str_t
-{
- const char *keywords;
- const char *district;// optional for expr_plus, otherwise set to NULL.
- enum MAAT_EXPR_TYPE expr_type;
- enum MAAT_MATCH_METHOD match_method;
- enum MAAT_CASE_TYPE hex_bin;
-};
-struct Maat_rgn_ip_t
-{
- enum MAAT_ADDR_TYPE addr_type;
- const char* src_ip;
- const char* mask_src_ip;
- const char* dst_ip;
- const char* mask_dst_ip;
- unsigned short src_port;
- unsigned short mask_src_port;
- unsigned short dst_port;
- unsigned short mask_dst_port;
- unsigned short protocol;
- enum MAAT_ADDR_DIRECTION direction;
-};
-struct Maat_rgn_ip_plus_t
-{
- enum MAAT_ADDR_TYPE addr_type;
-
- const char* saddr_format;//mask, range or CIDR
- const char* src_ip1;
- const char* src_ip2;
- const char* sport_format;//mask or range
- unsigned short src_port1;
- unsigned short src_port2;
-
- const char* daddr_format;//mask, range or CIDR
- const char* dst_ip1;
- const char* dst_ip2;
- const char* dport_format;//mask or range
- unsigned short dst_port1;
- unsigned short dst_port2;
-
- unsigned short protocol;
- enum MAAT_ADDR_DIRECTION direction;
-};
-struct Maat_rgn_intv_t
-{
- const char *district;// optional for expr_plus, otherwise set to NULL.
- unsigned int low_boundary;
- unsigned int up_boundary;
-};
-struct Maat_rgn_digest_t
-{
- unsigned long long orgin_len;
- const char* digest_string;
- short confidence_degree;
-};
-struct Maat_rgn_sim_t
-{
- char* target;
- short threshold;// 1~100
-};
-struct Maat_region_t
-{
- const char* table_name;
- int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number.
- enum MAAT_REGION_TYPE region_type;
- union
- {
- struct Maat_rgn_str_t expr_rule;
- struct Maat_rgn_ip_t ip_rule;
- struct Maat_rgn_intv_t interval_rule;
- struct Maat_rgn_digest_t digest_rule;
- struct Maat_rgn_sim_t similarity_rule;
- };
-};
-struct Maat_cmd_region
-{
- const char* table_name;
- int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number.
- enum MAAT_REGION_TYPE region_type;
- union
- {
- struct Maat_rgn_str_t expr_rule;
- struct Maat_rgn_ip_t ip_rule;
- struct Maat_rgn_ip_plus_t ip_plus_rule;
- struct Maat_rgn_intv_t interval_rule;
- struct Maat_rgn_digest_t digest_rule;
- struct Maat_rgn_sim_t similarity_rule;
- };
-};
-struct Maat_cmd_line
-{
- const char* table_name;
- const char* table_line;
- int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
- int label_id;
- int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
-};
-
-//Input string of REGION_EXPR and REGION_SIMILARITY need to be escapeed.
-char* Maat_str_escape(char* dst,int size,const char*src);
-
-
-//Returns number of successfully updated rule.
-//Return -1 for failed.
-int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op);
-int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op);
-int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op);
-
-//Return the value of key after the increment.
-//If the key does not exist, it is set to 0 before performing the operation.
-long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment);
-struct Maat_cmd_key
-{
- char* table_name;
- int rule_id;
-};
-void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number);
-int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys);
-int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size);
-int Maat_cmd_flushDB(Maat_feather_t feather);
-
-struct Maat_cmd_group2group
-{
- const char* table_name;
- int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
- int superior_group_id;
-};
-struct Maat_cmd_group2compile
-{
- const char* table_name;
- const char* virtual_table_name;
- int group_id;
- int compile_id;
- int clause_index;
- int not_flag;
-};
-
-int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id);
-int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g);
-int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c);
-
-//@param expire_after: expired after $expire_after$ seconds, set to 0 for never timeout.
-//@param label_id: bigger than 0 means this compile rule is to be indexed and quried by Maat_cmd_select; =0 not index
-int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after);
-
-struct Maat_command_batch;
-struct Maat_command_batch* Maat_command_batch_new(Maat_feather_t feather);
-
-int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id);
-int Maat_command_batch_set_group2group(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g);
-int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c);
-int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after);
-int Maat_command_batch_commit(struct Maat_command_batch* batch);
-
-int Maat_command_get_new_group_id(Maat_feather_t feather);
-int Maat_command_get_new_region_id(Maat_feather_t feather);
-
-#ifdef __cplusplus
-} //end extern"C"
-#endif
-
-
-#endif
-
diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h
deleted file mode 100644
index ab54f22..0000000
--- a/inc/Maat_rule.h
+++ /dev/null
@@ -1,353 +0,0 @@
-
-/*
-*****************Maat Deep Packet Inspection Policy Framework********
-* Maat is the Goddess of truth and justice in ancient Egyptian concept.
-* Her feather was the measure that determined whether the souls (considered
-* to reside in the heart) of the departed would reach the paradise of afterlife
-* successfully.
-* Author: contact@zhengchao.io
-* Version 2022-10-24 version 4.0.0
-*********************************************************
-*/
-#ifndef H_MAAT_RULE_H_INCLUDE
-#define H_MAAT_RULE_H_INCLUDE
-#ifdef __cplusplus
-extern "C"{
-#endif
-#include
-enum MAAT_CHARSET
-{
- CHARSET_NONE=0,
- CHARSET_GBK,
- CHARSET_BIG5,
- CHARSET_UNICODE,
- CHARSET_UTF8, // 4
- CHARSET_BIN, //5
- CHARSET_UNICODE_ASCII_ESC, // Unicode Escape format, prefix backslash-u hex, e.g. "\u627;"
- CHARSET_UNICODE_ASCII_ALIGNED,//Unicode Escape format, prefix backslash-u with 4 bytes aligned, e.g. "\u0627"
- CHARSET_UNICODE_NCR_DEC, //SGML Numeric character reference,decimal base, e.g. "ا"
- CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا"
- CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA
- CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD
- CHARSET_WINDOWS1251,
- __CHARSET_MAX
-};
-enum MAAT_ACTION
-{
- MAAT_ACTION_BLOCK=0,
- MAAT_ACTION_MONIT,
- MAAT_ACTION_WHITE
-};
-enum MAAT_POS_TYPE
-{
- MAAT_POSTYPE_EXPR=0,
- MAAT_POSTYPE_REGEX
-};
-typedef void* scan_status_t;
-typedef void* stream_para_t;
-typedef void* Maat_feather_t;
-
-
-#define MAX_SERVICE_DEFINE_LEN 128
-#define MAX_HUGE_SERVICE_DEFINE_LEN (1024*4)
-struct Maat_rule_t
-{
- int config_id;
- int service_id;
- unsigned char do_log;
- unsigned char do_blacklist;
- unsigned char action;
- unsigned char reserved;
- int serv_def_len;
- char service_defined[MAX_SERVICE_DEFINE_LEN];
-};
-#define MAAT_RULE_UPDATE_TYPE_FULL 1
-#define MAAT_RULE_UPDATE_TYPE_INC 2
-typedef void Maat_start_callback_t(int update_type,void* u_para);
-typedef void Maat_update_callback_t(int table_id,const char* table_line,void* u_para);
-typedef void Maat_finish_callback_t(void* u_para);
-
-
-
-
-
-//--------------------HITTING DETAIL DESCRIPTION BEGIN
-
-#define MAAT_MAX_HIT_RULE_NUM 8
-#define MAAT_MAX_EXPR_ITEM_NUM 8
-#define MAAT_MAX_HIT_POS_NUM 8
-#define MAAT_MAX_REGEX_GROUP_NUM 8
-
-//NOTE position buffer as hitting_regex_pos and hit_pos,are ONLY valid before next scan or Maat_stream_scan_string_end
-struct regex_pos_t
-{
- int group_num;
- int hitting_regex_len;
- const char* hitting_regex_pos;
- int grouping_len[MAAT_MAX_REGEX_GROUP_NUM];
- const char* grouping_pos[MAAT_MAX_REGEX_GROUP_NUM];
-};
-struct str_pos_t
-{
- int hit_len;
- const char* hit_pos;
-};
-struct sub_item_pos_t
-{
- enum MAAT_POS_TYPE ruletype;
- int hit_cnt;
- union
- {
- struct regex_pos_t regex_pos[MAAT_MAX_HIT_POS_NUM];
- struct str_pos_t substr_pos[MAAT_MAX_HIT_POS_NUM];
- };
-};
-
-struct Maat_region_pos_t
-{
-
- int region_id;
- int sub_item_num;
- struct sub_item_pos_t sub_item_pos[MAAT_MAX_EXPR_ITEM_NUM];
-};
-
-struct Maat_hit_detail_t
-{
- int config_id;//set <0 if half hit;
- int hit_region_cnt;
- struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM];
-};
-//--------------------HITTING DETAIL DESCRIPTION END
-
-//Abondon interface ,left for compatible.
-Maat_feather_t Maat_summon_feather(int max_thread_num,
- const char* table_info_path,
- const char* ful_cfg_dir,
- const char* inc_cfg_dir,
- void*logger);//MESA_handle_logger
-//Abondon interface ,left for compatible.
-Maat_feather_t Maat_summon_feather_json(int max_thread_num,
- const char* table_info_path,
- const char* json_rule,
- void* logger);
-
-Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger);
-int Maat_initiate_feather(Maat_feather_t feather);
-
-enum MAAT_INIT_OPT
-{
- MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger, SIZE=sizeof(int). DEFAULT:1,000 milliseconds.
- MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:60,000 milliseconds.
- MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
- MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
- MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
- MAAT_OPT_STAT_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
- MAAT_OPT_PERF_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
- MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT: no default.
- MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping.
- // 2 return hit pos and regex grouping pos;DEFAULT:0
- MAAT_OPT_INSTANCE_NAME, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1, no more than 11 bytes.DEFAULT: MAAT_$tableinfo_path$.
- MAAT_OPT_DECRYPT_KEY, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT.
- MAAT_OPT_REDIS_IP, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT.
- MAAT_OPT_REDIS_PORT, //VALUE is a unsigned short or a signed int, host order, SIZE= sizeof(unsigned short) or sizeof(int). No DEFAULT.
- MAAT_OPT_REDIS_INDEX, //VALUE is interger *, 0~15, SIZE=sizeof(int). DEFAULT: 0.
- MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1.
- MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF.
- MAAT_OPT_CUMULATIVE_UPDATE_OFF, //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE ON.
- MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old.
- //This option also disables background update.
- MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
- MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
- MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
- MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds.
- MAAT_OPT_JSON_IS_GZIPPED, //VALUE is NULL, SIZE is 0. Default: 0, Not compressed by gzip.
- MAAT_OPT_STATUS_OUTPUT_PROMETHEUS //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1.
-};
-//return -1 if failed, return 0 on success;
-int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
-enum MAAT_STATE_OPT
-{
- MAAT_STATE_VERSION=1, //Get current maat version, if maat is in update progress, the updating version is returned. VALUE is long long, SIZE=sizeof(long long).
- MAAT_STATE_LAST_UPDATING_TABLE, //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no
- MAAT_STATE_IN_UPDATING
-};
-int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size);
-
-void Maat_burn_feather(Maat_feather_t feather);
-
-//return table_id(>=0) if success,otherwise return -1;
-int Maat_table_register(Maat_feather_t feather,const char* table_name);
-//return 1 if success,otherwise return -1 incase invalid table_id or registed function number exceed 32;
-int Maat_table_callback_register(Maat_feather_t feather,short table_id,
- Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
- Maat_update_callback_t *update,//table line ,u_para
- Maat_finish_callback_t *finish,//u_para
- void* u_para);
-
-#define MAX_HIT_REGION_NUM_PER_GROUP 128
-
-struct Maat_hit_path_t
-{
- int Nth_scan;
- int region_id;
- int sub_group_id;
- int top_group_id;
- int virtual_table_id; // 0 is not a virtual table.
- int compile_id;
-};
-enum MAAT_SCAN_OPT
-{
- MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
- MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
- MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
- //Maat_get_scan_status returns actual got number.
-};
-//return 0 if success, return -1 when failed;
-int Maat_set_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, const void* value, int size);
-
-//return >=0 if success, return -1 when failed;
-int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size);
-
-//Return hit rule number, return -1 when error occurs,return -2 when hit current region
-//mid MUST set to NULL before fist call
-int Maat_scan_intval(Maat_feather_t feather,int table_id
- ,unsigned int intval
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num);
-int Maat_scan_addr(Maat_feather_t feather,int table_id
- ,struct ipaddr* addr
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num);
-int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
- ,struct ipaddr* addr,unsigned short int proto
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num);
-int Maat_full_scan_string(Maat_feather_t feather,int table_id
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int* found_pos,int rule_num
- ,scan_status_t* mid,int thread_num);
-//hit_detail could be NULL if not cared.
-int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
- ,int* detail_ret,scan_status_t* mid,int thread_num);
-
-stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num);
-int Maat_stream_scan_string(stream_para_t* stream_para
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int* found_pos,int rule_num
- ,scan_status_t* mid);
-//hited_detail could be NULL if not cared.
-int Maat_stream_scan_string_detail(stream_para_t* stream_para
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
- ,int* detail_ret,scan_status_t* mid);
-void Maat_stream_scan_string_end(stream_para_t* stream_para);
-
-stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num);
-int Maat_stream_scan_digest(stream_para_t* stream_para
- ,const char* data,int data_len,unsigned long long offset
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t* mid);
-void Maat_stream_scan_digest_end(stream_para_t* stream_para);
-
-int Maat_similar_scan_string(Maat_feather_t feather,int table_id
- ,const char* data,int data_len
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t* mid,int thread_num);
-
-void Maat_clean_status(scan_status_t* mid);
-
-typedef void* MAAT_RULE_EX_DATA;
-// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered.
-// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called.
-typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
- MAAT_RULE_EX_DATA* ad, long argl, void *argp);
-typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
- MAAT_RULE_EX_DATA* ad, long argl, void *argp);
-typedef void Maat_rule_EX_dup_func_t(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp);
-
-int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
- Maat_rule_EX_new_func_t* new_func,
- Maat_rule_EX_free_func_t* free_func,
- Maat_rule_EX_dup_func_t* dup_func,
- long argl, void *argp);
-//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data.
-MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
-
-//Sort rules by their evaluation order.
-//rule_array will be modified with sorted rule.
-//Return sortted rule number, maybe less than n_rule if some rules are invalid.
-size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_rule_t* rule_array, size_t n_rule);
-
-
-//Helper function for parsing space or tab seperated line.
-//Nth_column: the Nth column is numberd from 1.
-//Return 0 if success.
-int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len);
-
-
-//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table.
-typedef void* MAAT_PLUGIN_EX_DATA;
-typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
-typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);
-typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp);
-typedef int Maat_plugin_EX_key2index_func_t(const char* key);
-
-int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t* new_func,
- Maat_plugin_EX_free_func_t* free_func,
- Maat_plugin_EX_dup_func_t* dup_func,
- Maat_plugin_EX_key2index_func_t* key2index_func,
- long argl, void *argp);
-//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to FREE the data.
-MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key);
-
-
-int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t* new_func,
- Maat_plugin_EX_free_func_t* free_func,
- Maat_plugin_EX_dup_func_t* dup_func,
- long argl, void *argp);
-
-struct ip_address
-{
- int ip_type; //4: IPv4, 6: IPv6
- union
- {
- unsigned int ipv4; //network order
- unsigned int ipv6[4];
- };
-};
-
-int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
-
-int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t* new_func,
- Maat_plugin_EX_free_func_t* free_func,
- Maat_plugin_EX_dup_func_t* dup_func,
- long argl, void *argp);
-
-//Return order: Longger suffix first, then fqdn with bigger index first.
-int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
-
-int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t *new_func,
- Maat_plugin_EX_free_func_t *free_func,
- Maat_plugin_EX_dup_func_t *dup_func,
- long argl, void *argp);
-int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
-
-enum MAAT_RULE_OPT
-{
- MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size.
-};
-int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size);
-
-#ifdef __cplusplus
-}//end extern "C"
-#endif
-
-
-#endif // H_MAAT_RULE_H_INCLUDE
-
diff --git a/inc/bool_matcher.h b/inc/bool_matcher.h
deleted file mode 100644
index 25a0554..0000000
--- a/inc/bool_matcher.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- *
- * Copyright (c) 2018
- * String Algorithms Research Group
- * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
- * National Engineering Laboratory for Information Security Technologies (NELIST)
- * All rights reserved
- *
- * Written by: LIU YANBING (liuyanbing@iie.ac.cn)
- * Last modification: 2021-06-12
- *
- * This code is the exclusive and proprietary property of IIE-CAS and NELIST.
- * Usage for direct or indirect commercial advantage is not allowed without
- * written permission from the authors.
- *
- */
-
-#ifndef INCLUDE_BOOL_MATCHER_H
-#define INCLUDE_BOOL_MATCHER_H
-#include
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
- #define MAX_ITEMS_PER_BOOL_EXPR 8
-
- /* not_flag=0表示布尔项item_id必须出现；not_flag=1表示布尔项item_id不能出现 */
- struct bool_item
- {
- unsigned long long item_id;
- unsigned char not_flag;
- };
-
- /* At least one item's not_flag should be 0. */
- struct bool_expr
- {
- unsigned long long expr_id;
- void * user_tag;
- size_t item_num;
- struct bool_item items[MAX_ITEMS_PER_BOOL_EXPR];
- };
-
- struct bool_expr_match
- {
- unsigned long long expr_id;
- void * user_tag;
- };
-
- struct bool_matcher;
-
- struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size);
-
- /* Returned results are sorted by expr_id in descending order. */
- //Input item_ids MUST be ASCENDING order and NO duplication.
- int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result);
-
- void bool_matcher_free(struct bool_matcher * matcher);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/inc/gram_index_engine.h b/inc/gram_index_engine.h
deleted file mode 100644
index 452dc4d..0000000
--- a/inc/gram_index_engine.h
+++ /dev/null
@@ -1,72 +0,0 @@
-#ifndef _GRAM_INDEX_ENGINE_
-#define _GRAM_INDEX_ENGINE_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-enum GIE_operation
-{
- GIE_INSERT_OPT,
- GIE_DELETE_OPT
-};
-enum GIE_INPUT_FORMAT
-{
- GIE_INPUT_FORMAT_PLAIN,
- GIE_INPUT_FORMAT_SFH
-};
-
-typedef struct
-{
- /* data */
-}GIE_handle_t;
-
-
-typedef struct
-{
- unsigned int id;
- unsigned int sfh_length;//size of fuzzy_hash
- enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
- short cfds_lvl;
- char * sfh;
- void * tag;
-}GIE_digest_t;
-
-
-typedef struct
-{
- unsigned int id;
- short cfds_lvl;
- void * tag;
-}GIE_result_t;
-
-
-typedef struct
-{
- unsigned int gram_value;
- //unsigned int htable_num;
- unsigned int position_accuracy;
- enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
- //else id format==PALIN, means the input string is common string
- int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
-}GIE_create_para_t;
-
-
-GIE_handle_t * GIE_create(const GIE_create_para_t * para);
-
-
-int GIE_update(GIE_handle_t * handle, GIE_digest_t ** digests, int size);
-
-
-//return actual matched result count
-//return 0 when matched nothing;
-//return -1 when error occurs;
-int GIE_query(GIE_handle_t * handle, const char * data, int data_len, GIE_result_t * results, int result_size);
-
-void GIE_destory(GIE_handle_t * handle);
-int GIE_string_similiarity(const char *str1, int len1, const char *str2, int len2);
-int GIE_sfh_similiarity(const char *sfh1, int len1, const char *sfh2, int len2);
-
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/inc/stream_fuzzy_hash.h b/inc/stream_fuzzy_hash.h
deleted file mode 100644
index 9e85e81..0000000
--- a/inc/stream_fuzzy_hash.h
+++ /dev/null
@@ -1,78 +0,0 @@
-#ifndef _STREAM_FUZZY_HASH_
-#define _STREAM_FUZZY_HASH_
-
-/*
- * Copyright (C) MESA 2015
-
- *
- */
-
-#include
-#include
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TOTAL_LENGTH 0
-#define EFFECTIVE_LENGTH 1
-#define HASH_LENGTH 2
-
-// typedef sfh_instance_t void*;
-typedef struct
-{
-}sfh_instance_t;
-
-/**
- * create a fuzzy hash handle and return it.
- * @return [handle]
- */
-sfh_instance_t * SFH_instance(unsigned long long origin_len);
-
-/**
- * destroy context by a fuzzy hash handle.
- * @param handle [handle]
- */
-void SFH_release(sfh_instance_t * handle);
-
-/**
- * Feed the function your data.
- * Call this function several times, if you have several parts of data to feed.
- * @param handle [handle]
- * @param data [data that you want to fuzzy_hash]
- * @param size [data size]
- * @param offset [offset]
- * @return [return effective data length in current feed]
- */
-unsigned int SFH_feed(sfh_instance_t * handle, const char* data, unsigned int size, unsigned long long offset);
-
-/**
- * Obtain the fuzzy hash values.
- * @param handle [handle]
- * @param result [fuzzy hash result]
- * Fuzzy hash result with offsets(in the square brackets, with colon splitted).
- * eg. abc[1:100]def[200:300]
- * @param size [@result size]
- * @return [return zero on success, non-zero on error]
- */
-int SFH_digest(sfh_instance_t * handle, char* result, unsigned int size);
-
-/**
- * Obtain certain length of fuzzy hash status.
- * @param handle [handle]
- * @param type [length type]
- * TOTAL_LENGTH:Total length of data you have fed.
- * Overlapped data will NOT count for 2 times.
- * EFFECTIVE_LENGTH:Length of data that involved in the calculation of hash.
- * HASH_LENGTH:Hash result length.
- * @return [length value]
- */
-unsigned long long SFH_status(sfh_instance_t * handle, int type);
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
-
diff --git a/include/maat/maat.h b/include/maat/maat.h
new file mode 100644
index 0000000..5d17528
--- /dev/null
+++ b/include/maat/maat.h
@@ -0,0 +1,23 @@
+#pragma once
+struct maat_feather;
+/* network-order */
+struct ipv4_tuple4{
+ UINT32 saddr; /* network order */
+ UINT32 daddr; /* network order */
+ UINT16 source; /* network order */
+ UINT16 dest; /* network order */
+};
+
+#define IPV6_ADDR_LEN (sizeof(struct in6_addr))
+
+struct ipv6_tuple4
+{
+ UCHAR saddr[IPV6_ADDR_LEN] ;
+ UCHAR daddr[IPV6_ADDR_LEN] ;
+ UINT16 source; /* network order */
+ UINT16 dest; /* network order */
+};
+struct maat_scan_state;
+int maat_scan_ipv4(struct maat_feather *feather, struct maat_scan_state *state, const struct ipv4_tuple4 *tuple4, unsigned int *matched_ids, size_t n_match_id);
+int maat_scan_string(struct maat_feather *feather, struct maat_scan_state *state, const char *data, size_t length, )
+
diff --git a/lib/.gitignore b/lib/.gitignore
deleted file mode 100644
index 65043fb..0000000
--- a/lib/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-SI/
-*.I*
-*.P*
-*.S*
-*.W*
-*.[od]
-*.[1-9]*
-*.log
-*.a
diff --git a/readme.md b/readme.md
deleted file mode 100644
index 0394427..0000000
--- a/readme.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# 简介
-MAAT是古埃及神话中真理与正义女神，她的羽毛（feather）能够判断离世之人能否前往天堂。
-
-MAAT框架对网络流处理中的配置进行抽象，并具有配置写入、多机同步（基于Redis）、加载、扫描的功能。
-
-Maat_rule.h中描述了初始化和配置扫描的API。
-
-Maat_command.h中描述了配置写入的API。
-
-# 编译
-普通编译
-
-` Make`
-
-带调试符号编译
-
-`Make debug`
-
-安装
-
-`Make install`
-
-生成动态链接库 `./build/src/libmaatframe.so`
-
-生成静态链接库 `./build/src/libmaatframe.a`
\ No newline at end of file
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
deleted file mode 100644
index b880656..0000000
--- a/src/CMakeLists.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-cmake_minimum_required(VERSION 3.5)
-
-set(MAAT_FRAME_MAJOR_VERSION 3)
-set(MAAT_FRAME_MINOR_VERSION 6)
-set(MAAT_FRAME_PATCH_VERSION 0)
-set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
-
-message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
-
-add_definitions(-fPIC)
-set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table_schema.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
-include_directories(/opt/MESA/include/MESA/)
-
-# Static Library Output
-add_library(maat_frame_static STATIC ${MAAT_SRC})
-set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
-set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
-set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
-target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/)
-#target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis)
-target_link_libraries(maat_frame_static hiredis-static)
-target_link_libraries(maat_frame_static igraph-static)
-target_link_libraries(maat_frame_static ipmatcher-static)
-
-# Shared Library Output
-add_library(maat_frame_shared SHARED ${MAAT_SRC})
-set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX)
-set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe)
-set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1)
-set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION})
-set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION})
-set_target_properties(maat_frame_shared PROPERTIES LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/version.map")
-target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/)
-target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis)
-target_link_libraries(maat_frame_shared hiredis-static ${MAAT_DEPEND_DYN_LIB})
-target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB})
-target_link_libraries(maat_frame_shared ipmatcher-static)
-
-set(CMAKE_INSTALL_PREFIX /opt/MESA/)
-install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
-install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_command.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
-install(FILES ${PROJECT_SOURCE_DIR}/inc/stream_fuzzy_hash.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
-install(FILES ${PROJECT_SOURCE_DIR}/inc/gram_index_engine.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
-install(FILES ${PROJECT_SOURCE_DIR}/inc/bool_matcher.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
-install(TARGETS maat_frame_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ COMPONENT LIBRARIES)
diff --git a/src/entry/.gitignore b/src/entry/.gitignore
deleted file mode 100644
index 65043fb..0000000
--- a/src/entry/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-SI/
-*.I*
-*.P*
-*.S*
-*.W*
-*.[od]
-*.[1-9]*
-*.log
-*.a
diff --git a/src/entry/FQDN_engine.cpp b/src/entry/FQDN_engine.cpp
deleted file mode 100644
index d402f66..0000000
--- a/src/entry/FQDN_engine.cpp
+++ /dev/null
@@ -1,352 +0,0 @@
-/*
- *
- * Copyright (c) 2020
- * String Algorithms Research Group
- * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
- * National Engineering Laboratory for Information Security Technologies (NELIST)
- * All rights reserved
- *
- * Written by: LIU YANBING (liuyanbing@iie.ac.cn)
- * Last modification: 2020-09-01
- *
- * This code is the exclusive and proprietary property of IIE-CAS and NELIST.
- * Usage for direct or indirect commercial advantage is not allowed without
- * written permission from the authors.
- *
- */
-
-#include "FQDN_engine.h"
-#include
-#include
-#include
-#include
-#include
-
-/*************************************************************************************/
-
-//#include
-//#define popcnt_u64 _mm_popcnt_u64
-//Use gcc builtin function to replace SSE4.2 instruction for portability
-
-#ifdef _MSC_VER
-#include
-#define popcnt_u64 __popcnt
-#else
-#define popcnt_u64 __builtin_popcountl
-#endif
-
-#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++)
-
-struct packedRT_t
-{
- unsigned long long bitmap[4];
- unsigned int A;
- unsigned char B[4];
-};
-
-void * aligned_malloc(size_t size, size_t align)
-{
- void * malloc_ptr;
- void * aligned_ptr;
-
- /* Error if align is not a power of two. */
- if (align & (align - 1))
- {
- return ((void*) 0);
- }
-
- if (align==0 || size == 0)
- {
- return ((void *) 0);
- }
-
- malloc_ptr = malloc (sizeof(void *) + align - 1 + size);
- if (!malloc_ptr)
- {
- return ((void *) 0);
- }
-
- aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1));
-
- ((void **) aligned_ptr) [-1] = malloc_ptr;
-
- return aligned_ptr;
-}
-
-void aligned_free(void * aligned_ptr)
-{
- if (aligned_ptr)
- {
- free (((void **) aligned_ptr) [-1]);
- }
-}
-
-/*************************************************************************************/
-struct domain_impl_t
-{
- unsigned int id;
- int suf_match;
- unsigned int len;
- unsigned long long hash; /*64λϣֵΨһʾһ*/
- domain_impl_t * next;
- void * utag;
-};
-
-class CHashTrieFQDN
-{
-public:
- CHashTrieFQDN();
- ~CHashTrieFQDN();
-
- int initialize(const struct FQDN_rule * rules, size_t n_rule);
- int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result);
-
-protected:
- unsigned int rank(unsigned int h);
-
-protected:
- unsigned int m_num;
- domain_impl_t * m_domains;
- unsigned int m_H;
- unsigned int m_max_pat_len;
- unsigned long long * m_B;
- packedRT_t * m_RT;
- domain_impl_t ** m_matched;
- unsigned char m_case_tab[256];
-};
-
-
-const unsigned long long ONE=1;
-
-#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) )
-#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) )
-
-const unsigned long long A=6364136223846793005LL;
-
-CHashTrieFQDN::CHashTrieFQDN()
-{
- m_num=0;
- m_domains=NULL;
- m_B=NULL;
- m_RT=NULL;
- m_matched=NULL;
- FOR(c, 256) m_case_tab[c]=tolower(c);
-}
-
-CHashTrieFQDN::~CHashTrieFQDN()
-{
- if(m_domains!=NULL)
- {
- delete [] m_domains;
- }
-
- if(m_B!=NULL)
- {
- delete [] m_B;
- }
-
- if(m_RT!=NULL)
- {
- aligned_free(m_RT);
- }
-
- if(m_matched!=NULL)
- {
- delete [] m_matched;
- }
-}
-
-int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule)
-{
- long long mem_bytes=0;
-
- if(n_rule==0) return -1;
- m_num=n_rule;
- m_domains=new domain_impl_t[m_num];
- mem_bytes+=m_num*sizeof(domain_impl_t);
-
- unsigned int N=m_num;
- m_max_pat_len=0;
-
- FOR(k, m_num)
- {
- m_domains[k].id =rules[k].id;
- m_domains[k].suf_match=rules[k].is_suffix_match;
- m_domains[k].len=rules[k].len;
- m_domains[k].next=NULL;
- m_domains[k].utag=rules[k].user_tag;
-
- FOR(j, rules[k].len)
- {
- if(rules[k].FQDN[j]=='.') ++N;
- }
-
- if(m_max_pat_len>6];
- mem_bytes+=(m_H>>6)*sizeof(unsigned long long);
- FOR(i, (m_H>>6)) m_B[i]=0;
-
- m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64);
- mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t);
- FOR(i, (m_H>>8))
- {
- FOR(j, 4) m_RT[i].bitmap[j]=0;
- }
-
- FOR(k, m_num)
- {
- const unsigned char * pb=(const unsigned char *)rules[k].FQDN;
- m_domains[k].hash=0;
-
- for(int j=rules[k].len-1; j>=-1; --j)
- {
- if(j==-1 || pb[j]=='.')
- {
- unsigned int h=m_domains[k].hash&(m_H-1);
- set_bit(m_B, h);
- if(j==-1)
- {
- int q=h&255;
- m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63));
- }
- }
- else
- {
- m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]];
- }
- }
- }
-
- m_RT[0].A=0;
- FOR(i, (m_H>>8))
- {
- m_RT[i].B[0]=0;
- m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]);
- m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]);
- m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]);
- m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]);
- }
-
- int tn=m_RT[m_H>>8].A;
-
- m_matched=new domain_impl_t *[tn];
- mem_bytes+=tn*sizeof(domain_impl_t *);
- FOR(i, tn) m_matched[i]=NULL;
-
- FOR(k, m_num)
- {
- unsigned int h=m_domains[k].hash&(m_H-1);
- unsigned idx=rank(h);
- m_domains[k].next=m_matched[idx];
- m_matched[idx]=&(m_domains[k]);
- }
-
-// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20));
-
- return 1;
-}
-
-unsigned int CHashTrieFQDN::rank(unsigned int h)
-{
- int p=(h>>8);
- int r=((h&255)>>6);
- int s=(h&63);
- unsigned long long e=m_RT[p].bitmap[r]&((ONE<=-1; --j)
- {
- if(j==-1 || pb[j]=='.')
- {
- unsigned int h=hash&(m_H-1);
- if(is_bit_set(m_B, h)==0) break;
- HASH[t]=hash;
- P[t]=j+1;
- ++t;
- }
- else if(j+m_max_pat_len=0; t--)
- {
- unsigned int h=HASH[t]&(m_H-1);
- int q=h&255;
-
- if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63)))
- {
- unsigned idx=rank(h);
-
- for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next)
- {
- if(P[t]!=0 && pt->suf_match==0) continue;
- if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t])
- {
- //if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num;
- results[match_num].id=pt->id;
- results[match_num].offset=P[t];
- results[match_num].user_tag=pt->utag;
- ++match_num;
- if(match_num==n_result) return match_num;
- }
- }
- }
- }
-
- return match_num;
-}
-
-/*************************************************************************************/
-struct FQDN_engine
-{
- CHashTrieFQDN ht;
-};
-
-struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule)
-{
- struct FQDN_engine * instance=new struct FQDN_engine;
- if(instance->ht.initialize(rules, n_rule)<0)
- {
- delete instance;
- return NULL;
- }
- else
- {
- return instance;
- }
-}
-
-int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result)
-{
- if(instance==NULL) return -1;
- return instance->ht.search(FQDN, FQDN_len, results, n_result);
-}
-
-void FQDN_engine_free(struct FQDN_engine * instance)
-{
- if(instance!=NULL) delete instance;
-}
diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp
deleted file mode 100644
index ce226bd..0000000
--- a/src/entry/Maat_api.cpp
+++ /dev/null
@@ -1,2667 +0,0 @@
-#include
-#include
-#include
-#include
-#include
-
-#include "rulescan.h"
-#include "bool_matcher.h"
-#include "Maat_rule.h"
-#include "Maat_rule_internal.h"
-#include "Maat_utils.h"
-#include "Maat_garbage_collection.h"
-#include "Maat_hierarchy.h"
-#include "dynamic_array.h"
-#include "alignment_int64.h"
-#include "config_monitor.h"
-#include "map_str2int.h"
-#include "rulescan.h"
-#include "json2iris.h"
-
-
-inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
-{
- alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
- return;
-}
-inline void DEC_SCANNER_REF(Maat_scanner*scanner,int thread_num)
-{
-
- alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
- return;
-}
-
-inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
-{
- if( _mid &&
- _mid->is_last_scan==1 &&
- _mid->compile_mid &&
- Maat_hierarchy_compile_mid_has_NOT_clause(_mid->compile_mid))
- {
- return 1;
- }
- else
- {
- return 0;
- }
-}
-
-
-//return 1 if insert a unique id
-//return 0 if id is duplicated
-//return -1 if set is full
-int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned long long id)
-{
- size_t i=0;
- for(i=0; iscan_cnt++;
- return;
-}
-
-void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len)
-{
- memcpy(rule, rule_head, sizeof(struct Maat_rule_head));
- memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN));
- return;
-}
-struct compile_sort_para
-{
- double evaluation_order;
- int declared_clause_num;
- int compile_id;
- void* user;
-};
-static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_rule* compile_relation, void* user)
-{
- para->compile_id=compile_relation->compile_id;
- para->evaluation_order=compile_relation->evaluation_order;
- para->declared_clause_num=compile_relation->declared_clause_num;
- para->user=user;
- return;
-}
-static int compile_sort_para_compare(const struct compile_sort_para* a, const struct compile_sort_para* b)
-{
- //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority.
- if(a->evaluation_order!=0 && b->evaluation_order!=0)
- {
- if(a->evaluation_order - b->evaluation_order <0)
- {
- return -1;
- }
- else if(a->evaluation_order - b->evaluation_order >0)
- {
- return 1;
- }
- }
- //If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority.
- else if(a->evaluation_order + b->evaluation_order!= 0)
- {
- return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1;
- }
- //If compile rule's execute sequences are not specified or equal.
- if(a->declared_clause_num!=b->declared_clause_num)
- {
- return (a->declared_clause_num-b->declared_clause_num);
- }
- else
- {
- return (b->compile_id-a->compile_id);
- }
-
-}
-static int compare_compile_rule(const void *a, const void *b)
-{
- const struct Maat_compile_rule *ra=*(const struct Maat_compile_rule **)a;
- const struct Maat_compile_rule *rb=*(const struct Maat_compile_rule **)b;
-
- struct compile_sort_para sa, sb;
- compile_sort_para_set(&sa, ra, NULL);
- compile_sort_para_set(&sb, rb, NULL);
-
- return compile_sort_para_compare(&sa, &sb);
-}
-static int compile_sort_para_compare_no_type(const void* a, const void* b)
-{
- return compile_sort_para_compare((const struct compile_sort_para*) a, (const struct compile_sort_para*) b);
-
-}
-size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_rule_t* rule_array, size_t n_rule)
-{
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- struct compile_sort_para sort_para[n_rule];
- struct Maat_rule_t copy_rule_array[n_rule];
-
- memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule);
-
- struct Maat_compile_rule *p=NULL;
-
- size_t i=0, j=0;
- for(i=0; iscanner->hier, rule_array[i].config_id);
- if(p)
- {
- compile_sort_para_set(sort_para+j, p, copy_rule_array+j);
- j++;
- }
- }
- qsort(sort_para, j, sizeof(struct compile_sort_para),
- compile_sort_para_compare_no_type);
- for(i=0; in_wrapped_region=0;
- wraper->virtual_table_id=0;
- wraper->virtual_table_ids=NULL;
- for(i=0; i< n_rslt; i++)
- {
- region=(struct Maat_region_inner*)(rulescan_rslt[i].tag);
- if(region->district_id==district_id||district_id==DISTRICT_ANY)
- {
- wraper->wrapped_regions[wraper->n_wrapped_region]=region;
- wraper->n_wrapped_region++;
- }
- }
- wraper->is_last_region=is_last_region;
- wraper->virtual_table_id=virtual_table_id;
- wraper->Nth_scan=Nth_scan;
- wraper->virtual_table_ids=NULL;
- return;
-}
-void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* wraper, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virtual_table_id, int Nth_scan)
-{
- size_t i=0;
- wraper->n_wrapped_region=0;
- wraper->virtual_table_id=0;
- wraper->virtual_table_ids=NULL;
- for(i=0; i< n_rslt; i++)
- {
- wraper->wrapped_regions[wraper->n_wrapped_region]=(struct Maat_region_inner*)(GIE_rslt[i].tag);
- }
- wraper->n_wrapped_region=n_rslt;
- wraper->is_last_region=is_last_region;
- wraper->virtual_table_id=virtual_table_id;
- wraper->Nth_scan=Nth_scan;
- wraper->virtual_table_ids=NULL;
- return;
-}
-
-int region_compile(_Maat_feather_t*feather, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num)
-{
- int is_last_region=region_hit_wraper->is_last_region;
- size_t region_hit_num=region_hit_wraper->n_wrapped_region;
-
- int scan_ret=0;
- int i=0;
-
- struct Maat_compile_rule* compile_rule_array[size];
- struct Maat_compile_rule* compile_rule=NULL;
- int virtual_table_id=0;
-
- struct Maat_region_inner* region=NULL;
-
- for(i=0; (size_t)iwrapped_regions[i];
- assert(region->magic_num==REGION_RULE_MAGIC);
- if(region_hit_wraper->virtual_table_ids)
- {
- virtual_table_id=region_hit_wraper->virtual_table_ids[i];
- }
- else
- {
- virtual_table_id=region_hit_wraper->virtual_table_id;
- }
- Maat_hierarchy_compile_mid_udpate(feather->scanner->hier, compile_mid, region->region_id, virtual_table_id, region_hit_wraper->Nth_scan, i);
-
- }
- scan_ret=Maat_hierarchy_region_compile(feather->scanner->hier, compile_mid, is_last_region, (void**)compile_rule_array, size);
- //Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule.
- if(scan_ret>1)
- {
- qsort(compile_rule_array, scan_ret, sizeof(struct Maat_compile_rule*),
- compare_compile_rule);
- }
- for(i=0; imagic_num==COMPILE_RULE_MAGIC);
-
- fill_maat_rule(&(result[i]), &(compile_rule->head),
- compile_rule->service_defined, compile_rule->head.serv_def_len);
-
- }
-
- if(scan_ret>0)
- {
- alignment_int64_array_add(feather->hit_cnt, thread_num, 1);
- }
- if(region_hit_num==0&&scan_ret>0)
- {
- alignment_int64_array_add(feather->not_grp_hit_cnt, thread_num, 1);
-
- }
- return MIN(scan_ret, size);
-}
-
-
-int fill_regex_pos(struct regex_pos_t *regex_pos,int size,rule_result_t *rs_result,const char* buff)
-{
- int i=0,j=0;
- int group_num=rs_result->group_num;
- unsigned int * position=rs_result->position;
- unsigned int * length=rs_result->length;
- regex_pos->group_num=group_num;
- for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM)
- {
- break;
- }
- regex_pos[i].hitting_regex_pos=buff+position[(group_num+1)*i];
- regex_pos[i].hitting_regex_len=length[(group_num+1)*i];
- for(j=0;jMAAT_MAX_REGEX_GROUP_NUM)
- {
- break;
- }
- regex_pos[i].grouping_pos[j]=buff+position[(group_num+1)*i+j+1];
- regex_pos[i].grouping_len[j]=length[(group_num+1)*i+j+1];
- }
- }
- return i;
-}
-int fill_substr_pos(struct str_pos_t* str_pos,int size,rule_result_t *rs_result,const char* buff)
-{
- int i=0;
- unsigned int * position=rs_result->position;
- unsigned int * length=rs_result->length;
- for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM)
- {
- break;
- }
- str_pos[i].hit_pos=buff+position[i];
- str_pos[i].hit_len= length[i];
- }
- return i;
-}
-int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* rs_result,int rs_cnt,const char* buff)
-{
- int k=0;
- for(k=0;kNth_hit_region;
- hit_detail[i].region_pos[j].region_id=hit_paths[j]->path.region_id;
- hit_detail[i].region_pos[j].sub_item_num=region_hit[Nth_hit_region].rnum;
- hit_pos_RS2Maat(hit_detail[i].region_pos[j].sub_item_pos, MAAT_MAX_EXPR_ITEM_NUM,
- region_hit[Nth_hit_region].result, region_hit[Nth_hit_region].rnum, scan_buff);
- r_in_c_flag[Nth_hit_region]=1;
- }
- hit_detail[i].hit_region_cnt=j;
- }
- //for each region_ids, if belongs to no compile cfg, fill hit_detail as a half hit cfg
- for(j=0; j<(size_t)region_cnt && iregion_id;
- hit_detail[i].region_pos[0].sub_item_num=region_hit[j].rnum;
- hit_pos_RS2Maat(hit_detail[i].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM,
- region_hit[j].result, region_hit[j].rnum, scan_buff);
- i++;
- }
- }
- return i;
- */
- return 0;
-}
-
-struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num)
-{
- struct _OUTER_scan_status_t* outer_mid=NULL;
- outer_mid=ALLOC(struct _OUTER_scan_status_t, 1);
- outer_mid->feather=feather;
- outer_mid->district_id=DISTRICT_ANY;
- outer_mid->thread_num=(signed short)thread_num;
- return outer_mid;
-}
-struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid, _Maat_feather_t* feather, int thread_num, int is_hit_region)
-{
- struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*raw_mid);;
- if(_mid==NULL)
- {
- _mid=_make_outer_status(feather, thread_num);
- *raw_mid=_mid;
- if(_mid->thread_num>=0)//Maat_set_scan_status calls grap_mid() with thread_num=-1.
- {
- alignment_int64_array_add(feather->outer_mid_cnt, thread_num, 1);
- }
- }
- if(_mid->thread_num<0 && thread_num>=0)
- {
- _mid->thread_num=thread_num;
- alignment_int64_array_add(feather->outer_mid_cnt, thread_num, 1);
- }
-
- if(is_hit_region==1)
- {
- if(_mid->compile_mid==NULL)
- {
- _mid->compile_mid=Maat_hierarchy_compile_mid_new(feather->scanner->hier, thread_num);
- alignment_int64_array_add(feather->compile_mid_cnt, thread_num, 1);
- }
- }
- return _mid;
-}
-int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,int data_len, long* cache_offset)
-{
- int to_copy_size=0,foward_offset=0;
- int ret_len;
- if(data_len<=buff_size-detained_len)
- {
- to_copy_size=data_len;
- memcpy(buff+detained_len,data,data_len);
- ret_len=detained_len+data_len;
- }
- else if(data_len>buff_size-detained_len&&data_len=buff_size
- {
- memcpy(buff,data+data_len-buff_size,buff_size);
- ret_len=buff_size;
- *cache_offset+=(data_len-buff_size+detained_len);
- }
- return ret_len;
-}
-int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz)
-{
- int ret=0;
- struct stat fstat_buf;
- unsigned char* json_buff=NULL, *decrypted_buff=NULL, *uncompressed_buff=NULL;
- size_t json_buff_sz=0, decrypted_buff_sz=0, uncompressed_buff_sz=0;
-
- MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,
- "Maat initial with JSON file %s, formating..",
- maat_json_fn);
-
- if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo))
- {
- ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz);
- if(ret<0)
- {
- MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
- "Decrypt Maat JSON file %s failed.", maat_json_fn);
- return -1;
- }
- json_buff=decrypted_buff;
- json_buff_sz=decrypted_buff_sz;
-
- }
- if(feather->maat_json_is_gzipped)
- {
- ret=gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
- free(json_buff);
- if(ret<0)
- {
- MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
- "Uncompress Maat JSON file %s failed.", maat_json_fn);
- return -1;
- }
- json_buff=uncompressed_buff;
- json_buff_sz=uncompressed_buff_sz;
- }
- if(json_buff==NULL)//decryption failed or no decryption.
- {
- ret=load_file_to_memory(maat_json_fn, &json_buff, &json_buff_sz);
- if(ret<0)
- {
- MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module,
- "Read Maat JSON file %s failed.", maat_json_fn);
- return -1;
- }
- }
- ret=json2iris((const char*)json_buff,
- maat_json_fn,
- feather->compile_tn, feather->group2compile_tn, feather->group2group_tn,
- NULL,
- feather->json_ctx.iris_file,
- sizeof(feather->json_ctx.iris_file),
- strlen(feather->decrypt_key)?feather->decrypt_key:NULL,
- strlen(feather->decrypt_algo)?feather->decrypt_algo:NULL,
- feather->logger);
- free(json_buff);
- json_buff=NULL;
- if(ret<0)
- {
- return -1;
- }
- if(!feather->is_running)
- {
- strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file));
- }
-
- ret=stat(maat_json_fn, &fstat_buf);
- feather->json_ctx.last_md5_time=fstat_buf.st_ctim;
-
- md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5);
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module,
- "JSON file %s md5: %s, generate index file %s OK.",
- feather->json_ctx.json_file,
- feather->json_ctx.effective_json_md5,
- feather->json_ctx.iris_file);
- feather->input_mode=SOURCE_JSON_FILE;
-
- return 0;
-}
-Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger)
-{
- if(max_thread_num<=0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Invalid max_thread_num=%d."
- ,max_thread_num);
- return NULL;
- }
- _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1);
- feather->table_mgr=Maat_table_manager_create(table_info_path, logger);
- if(feather->table_mgr==NULL)
- {
- goto failed;
- }
- Maat_table_manager_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn));
- Maat_table_manager_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn));
- Maat_table_manager_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn));
-
- feather->logger=logger;
- feather->scan_thread_num=max_thread_num;
-
- feather->rule_effect_interval_ms=60*1000;
- feather->rule_update_checking_interval_ms=1*1000;
- feather->garbage_collection_timeout_ms=10*1000;
- feather->rule_scan_type=0;
- feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num);
- feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
- feather->compile_mid_cnt=alignment_int64_array_alloc(max_thread_num);
- feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
- feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
- feather->maat_version=0;
- feather->last_full_version=0;
- feather->base_grp_seq=0;
- feather->base_rgn_seq=0;
- feather->AUTO_NUMBERING_ON=1;
- feather->backgroud_update_enabled=1;
- feather->maat_json_is_gzipped=0;
-
- snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc");
- snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path);
- pthread_mutex_init(&(feather->background_update_mutex),NULL);
- snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
- return feather;
-failed:
- free(feather);
- return NULL;
-}
-int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
-{
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- int intval=0,ret=-1;
- char err_str[MAX_TABLE_NAME_LEN];
- switch(type)
- {
- case MAAT_OPT_ENABLE_UPDATE:
- intval=*((int*)value);
- if(_feather->backgroud_update_enabled!=intval)
- {
- if(intval==0)
- {
- pthread_mutex_lock(&(_feather->background_update_mutex));
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Background update is disabled, current version %lld."
- ,_feather->maat_version);
- }
- else
- {
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Background update is enabled, current version %lld."
- ,_feather->maat_version);
- }
- _feather->backgroud_update_enabled=intval;
- }
- else
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Duplicated operation, background update is ALREADY %s, current version %lld."
- ,_feather->backgroud_update_enabled?"enabled":"disabled"
- ,_feather->maat_version);
-
- }
- return 0;
- default:
- break;
- }
- if(_feather->is_running==1)//The following options are not allowed to set after initiation;
- {
- return -2;
- }
- switch(type)
- {
- case MAAT_OPT_EFFECT_INVERVAL_MS:
- intval=*(const int*)value;
- if(size!=sizeof(int)||intval<0)
- {
- return -1;
- }
- _feather->rule_effect_interval_ms=intval;
- break;
- case MAAT_OPT_SCANDIR_INTERVAL_MS:
- intval=*(const int*)value;
- if(size!=sizeof(int)||intval<0)
- {
- return -1;
- }
- _feather->rule_update_checking_interval_ms=intval;
- break;
- case MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS:
- intval=*(const int*)value;
- if(size!=sizeof(int)||intval<0)
- {
- return -1;
- }
- _feather->garbage_collection_timeout_ms=intval;
- break;
- case MAAT_OPT_STATUS_OUTPUT_PROMETHEUS:
- intval=*(const int*)value;
- if(size!=sizeof(int)||intval<0)
- {
- return -1;
- }
- _feather->output_prometheus=intval;
- break;
- case MAAT_OPT_JSON_IS_GZIPPED:
- intval=*(const int*)value;
- if(size!=sizeof(int)||intval<0)
- {
- return -1;
- }
- if(intval>0)
- {
- _feather->maat_json_is_gzipped=1;
- }
- else
- {
- _feather->maat_json_is_gzipped=0;
- }
- break;
- case MAAT_OPT_FULL_CFG_DIR:
- assert(_feather->input_mode==SOURCE_NONE);
- if(size>(int)sizeof(_feather->iris_ctx.full_dir))
- {
- return -1;
- }
- memcpy(_feather->iris_ctx.full_dir,(const char*)value,size);
- _feather->input_mode=SOURCE_IRIS_FILE;
- break;
- case MAAT_OPT_INC_CFG_DIR:
- if(size>(int)sizeof(_feather->iris_ctx.inc_dir))
- {
- return -1;
- }
- memcpy(_feather->iris_ctx.inc_dir,(const char*)value,size);
- break;
- case MAAT_OPT_JSON_FILE_PATH:
- assert(_feather->input_mode==SOURCE_NONE);
- ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str));
- if(ret<0)
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module,
- "Load maat json file %s failed: %s.",
- (const char*)value, err_str);
- return -1;
- }
- break;
- case MAAT_OPT_STAT_ON:
- _feather->stat_on=1;
- break;
- case MAAT_OPT_PERF_ON:
- _feather->perf_on=1;
- break;
- case MAAT_OPT_STAT_FILE_PATH:
- if(size>(int)sizeof(_feather->stat_file))
- {
- return -1;
- }
- memcpy(_feather->stat_file, (const char*)value, size);
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module,
- "Maat performance statistic output to %s.",
- (const char*)value);
- _feather->stat_on=1;
- break;
- case MAAT_OPT_SCAN_DETAIL:
- intval=*(const int*)value;
- _feather->rule_scan_type=intval;
- break;
- case MAAT_OPT_INSTANCE_NAME:
- snprintf(_feather->instance_name,
- sizeof(_feather->instance_name),
- "%s",
- (const char*)value);
- break;
- case MAAT_OPT_DECRYPT_KEY:
- if((size_t)size>sizeof(_feather->decrypt_key))
- {
- return -1;
- }
- memcpy(_feather->decrypt_key,value,size);
- break;
- case MAAT_OPT_REDIS_IP:
- assert(_feather->input_mode==SOURCE_NONE);
- if((size_t)size>sizeof(_feather->mr_ctx.redis_ip))
- {
- return -1;
- }
- memcpy(_feather->mr_ctx.redis_ip,value,size);
- _feather->input_mode=SOURCE_REDIS;
- break;
- case MAAT_OPT_REDIS_PORT:
- if((size_t)size==sizeof(unsigned short))
- {
- _feather->mr_ctx.redis_port=*((unsigned short*)value);
- }
- else if((size_t)size==sizeof(int))
- {
- _feather->mr_ctx.redis_port=*((int*)value);
- }
- else
- {
- return -1;
- }
- break;
- case MAAT_OPT_REDIS_INDEX:
- if((size_t)size!=sizeof(int))
- {
- return -1;
- }
- _feather->mr_ctx.redis_db=*((int*)value);
- break;
- case MAAT_OPT_CMD_AUTO_NUMBERING:
- if((size_t)size!=sizeof(int)||*((int*)value)>15||*((int*)value)<0)
- {
- return -1;
- }
- _feather->AUTO_NUMBERING_ON=*((int*)value);
- break;
- case MAAT_OPT_DEFERRED_LOAD:
- _feather->DEFERRED_LOAD_ON=1;
- break;
- case MAAT_OPT_CUMULATIVE_UPDATE_OFF:
- _feather->cumulative_update_off=1;
- break;
- case MAAT_OPT_LOAD_VERSION_FROM:
- _feather->load_version_from=*((long long*)value);
- _feather->backgroud_update_enabled=1;
- pthread_mutex_lock((&_feather->background_update_mutex));
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Maat load version from %lld, backgroud update stopped."
- ,_feather->load_version_from);
- break;
- case MAAT_OPT_ACCEPT_TAGS:
- _feather->n_tags=parse_accept_tag((const char*) value, &_feather->accept_tags, _feather->logger);
- if(_feather->n_tags==0)
- {
- return -1;
- }
- _feather->accept_tags_raw=_maat_strdup((const char*) value);
- break;
- case MAAT_OPT_FOREIGN_CONT_DIR:
- memset(_feather->foreign_cont_dir, 0, sizeof(_feather->foreign_cont_dir));
- strncpy(_feather->foreign_cont_dir, (char*)value, sizeof(_feather->foreign_cont_dir));
- if(_feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]=='/')
- {
- _feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]='\0';
- }
- break;
- default:
- return -1;
- }
- return 0;
-}
-void maat_read_full_config(_Maat_feather_t* _feather)
-{
- struct source_redis_ctx* mr_ctx=NULL;
- switch(_feather->input_mode)
- {
- case SOURCE_REDIS:
- mr_ctx=&(_feather->mr_ctx);
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Maat initiate from Redis %s:%hu db%d."
- ,mr_ctx->redis_ip
- ,mr_ctx->redis_port
- ,mr_ctx->redis_db);
- mr_ctx->read_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, _feather->logger);
- if(mr_ctx->read_ctx != NULL)
- {
- redis_monitor_traverse(_feather->maat_version,
- mr_ctx,
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- _feather,
- _feather->decrypt_key, //Not used.
- _feather);
- }
- if(_feather->update_tmp_scanner==NULL)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
- "At initiation: no avilable rule in redis %s:%hu db%d",
- mr_ctx->redis_ip,
- mr_ctx->redis_port,
- mr_ctx->redis_db);
- }
- break;
- case SOURCE_IRIS_FILE:
- config_monitor_traverse(_feather->maat_version,
- _feather->iris_ctx.full_dir,
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- _feather,
- _feather->decrypt_key,
- _feather->logger);
- if(_feather->update_tmp_scanner==NULL)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
- "At initiation: NO effective rule in %s.",
- _feather->iris_ctx.full_dir);
- }
- break;
- case SOURCE_JSON_FILE:
- config_monitor_traverse(_feather->maat_version,
- _feather->json_ctx.iris_file,
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- _feather,
- _feather->decrypt_key,
- _feather->logger);
- if(_feather->update_tmp_scanner==NULL)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
- "At initiation: NO efffective rule in JSON generate %s.",
- _feather->json_ctx.iris_file);
- }
- break;
- default:
- break;
- }
- _feather->scanner=_feather->update_tmp_scanner;
- _feather->update_tmp_scanner=NULL;
- _feather->is_running=1;
- if(_feather->scanner!=NULL)
- {
- _feather->maat_version=_feather->scanner->version;
- _feather->last_full_version=_feather->scanner->version;
- }
- return;
-}
-int Maat_initiate_feather(Maat_feather_t feather)
-{
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- system_cmd_mkdir(_feather->foreign_cont_dir);
- _feather->garbage_bin=Maat_garbage_bin_new(_feather->rule_effect_interval_ms/1000+_feather->garbage_collection_timeout_ms/1000);
-
- if(_feather->DEFERRED_LOAD_ON==0)
- {
- maat_read_full_config(_feather);
- }
- if(strlen(_feather->stat_file)==0)
- {
- if(_feather->stat_on==1)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
- "At initiation: MAAT_OPT_STAT_FILE_PATH not set, TURN OFF STAT trigger.");
- }
- _feather->stat_on=0;
- }
- if(_feather->stat_on==0&&_feather->perf_on==1)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
- "At initiation: STAT tirigger OFF, TURN OFF PERF trigger.");
- _feather->perf_on=0;
- }
- if(_feather->cumulative_update_off==1)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Update with cumulative version OFF.");
- }
- if(_feather->n_tags>0)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
- "Accept tags: %s", _feather->accept_tags_raw);
- }
- if(_feather->stat_on==1)
- {
- maat_stat_init(_feather);
- }
-
- pthread_create(&(_feather->cfg_mon_t), NULL, thread_rule_monitor, (void*)_feather);
- return 0;
-
-}
-
-Maat_feather_t Maat_summon_feather(int max_thread_num,
- const char* table_info_path,
- const char* ful_cfg_dir,
- const char* inc_cfg_dir,
- void* logger)
-{
- int ret=-1;
- Maat_feather_t feather=NULL;
- feather=Maat_feather(max_thread_num,table_info_path,logger);
- if(feather==NULL)
- {
- return NULL;
- }
- ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1);
- if(ret<0)
- {
- goto error_out;
- }
- ret=Maat_set_feather_opt(feather, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1);
- if(ret<0)
- {
- goto error_out;
- }
- ret=Maat_initiate_feather(feather);
- if(ret<0)
- {
- goto error_out;
- }
- return feather;
-
-error_out:
- Maat_burn_feather(feather);
- return NULL;
-
-
-}
-Maat_feather_t Maat_summon_feather_json(int max_thread_num,
- const char* table_info_path,
- const char* json_rule,
- void* logger)
-{
- int ret=-1;
- Maat_feather_t feather=NULL;
- feather=Maat_feather(max_thread_num,table_info_path,logger);
- if(feather==NULL)
- {
- return NULL;
- }
- ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_rule, strlen(json_rule)+1);
- if(ret<0)
- {
- goto error_out;
- }
- ret=Maat_initiate_feather(feather);
- if(ret<0)
- {
- goto error_out;
- } return feather;
-
-error_out:
-
- Maat_burn_feather(feather);
- return NULL;
-
-}
-
-void Maat_burn_feather(Maat_feather_t feather)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- _feather->is_running=0;//destroy will proceed in thread_rule_monitor
- void* ret=NULL;
- pthread_join(_feather->cfg_mon_t, &ret);
- return;
-}
-int Maat_table_register(Maat_feather_t feather,const char* table_name)
-{
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- return Maat_table_manager_get_id_by_name(_feather->table_mgr, table_name);
-}
-int Maat_table_callback_register(Maat_feather_t feather,short table_id,
- Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
- Maat_update_callback_t *update,//table line ,u_para
- Maat_finish_callback_t *finish,//u_para
- void* u_para)
-{
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- int i=0,ret=0;
-
- pthread_mutex_lock(&(_feather->background_update_mutex));
- ret=Maat_table_manager_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para);
- if(ret<0)
- {
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- return -1;
- }
- if(!_feather->scanner)
- {
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- return 1;
- }
- const char* line=NULL;
- struct Maat_table_runtime* table_rt=NULL;
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- long long line_cnt=Maat_table_runtime_plugin_cached_row_count(table_rt);
-
- if(line_cnt>0)
- {
- if(start!=NULL)
- {
- start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
- }
- for(i=0; ibackground_update_mutex));
- return 1;
-}
-
-void rule_ex_data_new_cb(void * user_data, void * param)
-{
- struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)param;
- struct Maat_compile_rule* compile=(struct Maat_compile_rule*)user_data;
- MAAT_RULE_EX_DATA ad=NULL;
-
- if(compile->ref_table->table_id!=ex_desc->table_id)
- {
- return;
- }
- ad=rule_ex_data_new(&(compile->head),
- compile->service_defined,
- ex_desc);
- compile->ads[ex_desc->idx]=ad;
- return;
-}
-int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
- Maat_rule_EX_new_func_t *new_func,
- Maat_rule_EX_free_func_t* free_func,
- Maat_rule_EX_dup_func_t* dup_func,
- long argl, void *argp)
-{
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- int idx=-1;
-
- if(new_func==NULL || free_func==NULL || dup_func==NULL)
- {
- return -1;
- }
- pthread_mutex_lock(&(_feather->background_update_mutex));
- idx=Maat_table_manager_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name,
- new_func,
- free_func,
- dup_func,
- argl, argp);
-
- if(idx<0)
- {
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- return -1;
- }
- struct compile_ex_data_idx* compile_ex_desc=Maat_table_manager_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx);
-
- if(_feather->scanner!=NULL)
- {
- Maat_hierarchy_compile_user_data_iterate(_feather->scanner->hier, rule_ex_data_new_cb, compile_ex_desc);
- }
-
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- return idx;
-
-}
-MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
-{
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- struct Maat_compile_rule* compile=NULL;
- const struct compile_table_schema* compile_desc=NULL;
- const struct compile_ex_data_idx* ex_desc=NULL;
- MAAT_RULE_EX_DATA ad=NULL;
- if(!_feather || !_feather->scanner)
- {
- return NULL;
- }
- compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id);
- if(compile==NULL)
- {
- return NULL;
- }
- compile_desc=&(compile->ref_table->compile);
- assert(idxex_data_num);
- ex_desc=compile_desc->ex_desc+idx;
- ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp);
- return ad;
-}
-
-
-MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_table_schema *table_desc=NULL;
- struct Maat_table_runtime *table_rt=NULL;
- MAAT_RULE_EX_DATA exdata=NULL;
- struct timespec start,end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
-
- if(_feather->scanner==NULL)
- {
- return NULL;
- }
- table_desc=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
- }
-
- return exdata;
-}
-int Maat_generic_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t *new_func,
- Maat_plugin_EX_free_func_t *free_func,
- Maat_plugin_EX_dup_func_t *dup_func,
- long argl, void *argp)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- int ret=-1;
- struct Maat_table_schema *table_schema=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id);
- pthread_mutex_lock(&(_feather->background_update_mutex));
- ret=Maat_table_schema_set_EX_data_schema(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
-
- if(ret<0)
- {
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- return -1;
- }
-
- struct Maat_table_runtime* table_rt=NULL;
- if(_feather->scanner!=NULL)
- {
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- Maat_table_runtime_commit_EX_data_schema(table_rt, table_schema, _feather->logger);
- }
- pthread_mutex_unlock(&(_feather->background_update_mutex));
-
- return 0;
-
-}
-int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t* new_func,
- Maat_plugin_EX_free_func_t* free_func,
- Maat_plugin_EX_dup_func_t* dup_func,
- Maat_plugin_EX_key2index_func_t* key2index_func,
- long argl, void *argp)
-
-{
- int ret=-1;
- ret=Maat_generic_plugin_EX_register(feather, table_id,
- new_func, free_func, dup_func,
- argl, argp);
- return ret;
-
-}
-
-int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t *new_func,
- Maat_plugin_EX_free_func_t *free_func,
- Maat_plugin_EX_dup_func_t *dup_func,
- long argl, void *argp)
-
-{
- int ret=-1;
- ret=Maat_generic_plugin_EX_register(feather, table_id,
- new_func, free_func, dup_func,
- argl, argp);
- return ret;
-}
-int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t *new_func,
- Maat_plugin_EX_free_func_t *free_func,
- Maat_plugin_EX_dup_func_t *dup_func,
- long argl, void *argp)
-{
- int ret=-1;
- ret=Maat_generic_plugin_EX_register(feather, table_id,
- new_func, free_func, dup_func,
- argl, argp);
- return ret;
-}
-int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id,
- Maat_plugin_EX_new_func_t *new_func,
- Maat_plugin_EX_free_func_t *free_func,
- Maat_plugin_EX_dup_func_t *dup_func,
- long argl, void *argp)
-{
- int ret=-1;
- ret=Maat_generic_plugin_EX_register(feather, table_id,
- new_func, free_func, dup_func,
- argl, argp);
- return ret;
-}
-
-
-int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
-{
- struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
- struct Maat_table_schema *table_schema=NULL;
- struct Maat_table_runtime *table_rt=NULL;
- int n_get=0;
-
- struct timespec start, end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- if(_feather->scanner==NULL)
- {
- return 0;
- }
-
- table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
- if(table_type!=TABLE_TYPE_IP_PLUGIN)
- {
- return -1;
- }
- struct ip_data ip_data=*(const struct ip_data*)ip;
- if(ip_data.type==IPv4)
- {
- ip_data.ipv4=ntohl(ip_data.ipv4);
- }
- else
- {
- ipv6_ntoh(ip_data.ipv6);
- }
- n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data);
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
- }
-
- return n_get;
-
-}
-int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
-{
- struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
- struct Maat_table_schema *table_schema=NULL;
- struct Maat_table_runtime *table_rt=NULL;
- int n_get=0;
-
- struct timespec start, end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- if(_feather->scanner==NULL)
- {
- return 0;
- }
- table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
- if(table_type!=TABLE_TYPE_FQDN_PLUGIN)
- {
- return -1;
- }
- n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data);
-
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
- }
-
- return n_get;
-}
-int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
-{
- struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather;
- struct Maat_table_schema *table_schema=NULL;
- struct Maat_table_runtime *table_rt=NULL;
- int n_get=0;
-
- struct timespec start, end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- if(_feather->scanner==NULL)
- {
- return 0;
- }
- table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_BOOL_PLUGIN, NULL);
- table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
- enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
- if(table_type!=TABLE_TYPE_BOOL_PLUGIN)
- {
- return -1;
- }
- n_get=Maat_table_runtime_bool_plugin_get_N_ex_data(table_rt, table_schema, item_ids, n_item, ex_data_array, n_ex_data);
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
- }
-
- return n_get;
-}
-
-int Maat_full_scan_string_detail(Maat_feather_t feather, int table_id,
- enum MAAT_CHARSET charset, const char* data, int data_len,
- struct Maat_rule_t*result, int rule_num, struct Maat_hit_detail_t *hit_detail, int detail_num,
- int* detail_ret, scan_status_t* mid, int thread_num)
-{
- int region_ret=0, compile_ret=0;
- int district_id=DISTRICT_ANY;
- size_t hit_region_cnt=0;
- unsigned int sub_type=0;
- int virtual_table_id=0;
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid);
-
- scan_result_t *region_result=NULL;
- struct Maat_table_schema *p_table=NULL;
- struct string_table_schema* expr_desc=NULL;
- struct timespec start,end;
- Maat_scanner* my_scanner=NULL;
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- _mid=grab_mid(mid, _feather, thread_num, 0);
- scan_staus_count_inc(_mid);
- if(data==NULL||data_len<=0)
- {
- return -1;
- }
- my_scanner=_feather->scanner;
- if(my_scanner==NULL)
- {
- return 0;
- }
-
- p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
- if(p_table==NULL)
- {
- _feather->scan_err_cnt++;
- return -1;
- }
- expr_desc=&(p_table->expr);
-
- if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
- {
- _feather->scan_err_cnt++;
- return -1;
- }
- if(expr_desc->do_charset_merge==1)
- {
- sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0);
- }
- else
- {
- sub_type=make_sub_type(p_table->table_id, charset,0);
- }
- alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
- scan_data_t scan_data;
- scan_data.text_data.text=data;
- scan_data.text_data.tlen=data_len;
- scan_data.text_data.toffset=0;
-
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
-// Even no rule in table, we still need to search for NOT compile.
-// if(table_rt->origin_rule_num==0)
-// {
-// return 0;
-// }
- assert(thread_num<_feather->scan_thread_num);
- region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
-
- INC_SCANNER_REF(my_scanner, thread_num);
- if(table_rt->expr.expr_rule_cnt>0)
- {
- scan_data.rule_type=RULETYPE_STR;
- scan_data.sub_type=sub_type;
- region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result, MAX_SCANNER_HIT_NUM);
- if(region_ret>0)
- {
- hit_region_cnt+=region_ret;
- }
- }
- if(table_rt->expr.regex_rule_cnt>0)
- {
- scan_data.rule_type=RULETYPE_REG;
- scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0);
- region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt);
- if(region_ret>0)
- {
- hit_region_cnt+=region_ret;
- }
-
- }
-
- if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
- {
- struct scan_region_hit_wraper region_hit_wraper;
- if(hit_region_cnt>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
- }
- _mid=grab_mid(mid, _feather, thread_num, 1);
- if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
- {
- district_id=_mid->district_id;
- }
- scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, district_id,
- _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
-
- if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid))
- {
- compile_ret=region_compile(_feather, _mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0)
- {
- *detail_ret=fill_region_hit_detail(data, _mid->compile_mid,
- region_result, hit_region_cnt,
- result, compile_ret,
- _mid->scan_cnt,
- hit_detail, detail_num, my_scanner);
- }
- }
- }
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, thread_num);
- }
- DEC_SCANNER_REF(my_scanner, thread_num);
- if(compile_ret==0&&hit_region_cnt>0)
- {
- return -2;
- }
- return compile_ret;
-}
-int Maat_full_scan_string(Maat_feather_t feather, int table_id,
- enum MAAT_CHARSET charset, const char* data, int data_len,
- struct Maat_rule_t*result, int* found_pos, int rule_num,
- scan_status_t* mid, int thread_num)
-{
- int detail_ret=0,compile_ret=0;
- compile_ret=Maat_full_scan_string_detail(feather,table_id,
- charset, data,data_len,
- result, rule_num,
- NULL, 0, &detail_ret,mid,thread_num);
- return compile_ret;
-}
-int Maat_scan_intval(Maat_feather_t feather,int table_id
- ,unsigned int intval
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num)
-{
- int region_ret=0,compile_ret=0;
- int district_id=DISTRICT_ANY;
- struct _OUTER_scan_status_t* _mid=NULL;
- scan_data_t intval_scan_data;
- scan_result_t *region_result=NULL;
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_scanner* my_scanner=NULL;
-
- Maat_table_schema* p_table=NULL;
- struct timespec start,end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- _mid=grab_mid(mid, _feather, thread_num, 0);
- scan_staus_count_inc(_mid);
-
- int virtual_table_id=0;
- p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id);
- if(p_table==NULL)
- {
- _feather->scan_err_cnt++;
- return -1;
- }
- my_scanner=_feather->scanner;
- if(my_scanner==NULL)
- {
- return 0;
- }
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
- if(table_rt->origin_rule_num>0) // Even no rule in table, we still need to search for NOT compile.
- {
- if(p_table->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
- {
- _feather->scan_err_cnt++;
- return -1;
- }
-
- intval_scan_data.rule_type=RULETYPE_INT;
- intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0);
- intval_scan_data.int_data=intval;
-
- alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
-
- region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
-
- INC_SCANNER_REF(my_scanner,thread_num);
- region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM);
- if(region_ret<0)
- {
- DEC_SCANNER_REF(my_scanner, thread_num);
- _feather->scan_err_cnt++;
- return -1;
- }
- }
-
- if(region_ret>0 || scan_status_should_compile_NOT(_mid))
- {
- struct scan_region_hit_wraper region_hit_wraper;
- if(region_ret>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
- }
- _mid=grab_mid(mid, _feather, thread_num, 1);
- if(region_ret>0&&p_table->table_type==TABLE_TYPE_INTERVAL_PLUS)
- {
- district_id=_mid->district_id;
- }
-
- scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, district_id,
- _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
-
- if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid))
- {
- compile_ret=region_compile(_feather,_mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- }
- }
-
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num);
- }
- DEC_SCANNER_REF(my_scanner,thread_num);
-
- if(compile_ret==0&®ion_ret>0)
- {
- return -2;
- }
- return compile_ret;
-
-}
-
-int Maat_similar_scan_string(Maat_feather_t feather, int table_id,
- const char* data, int data_len,
- struct Maat_rule_t*result, int rule_num,
- scan_status_t* mid, int thread_num)
-{
- int hit_region_cnt=0,compile_ret=0;
- struct _OUTER_scan_status_t* _mid=NULL;
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_scanner* my_scanner=NULL;
- Maat_table_schema* p_table=NULL;
- struct timespec start,end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- _mid=grab_mid(mid, _feather, thread_num, 0);
- scan_staus_count_inc(_mid);
-
- int virtual_table_id=0;
- p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
- if(p_table==NULL)
- {
- _feather->scan_err_cnt++;
- return -1;
- }
- if(p_table->table_type!=TABLE_TYPE_SIMILARITY)
- {
- return -1;
- }
-
- my_scanner=_feather->scanner;
- if(my_scanner==NULL)
- {
- return 0;
- }
- GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
-
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id);
-
- GIE_handle_t* gie_handle=table_rt->similar.gie_handle;
- if(gie_handle==NULL)
- {
- return 0;
- }
- INC_SCANNER_REF(my_scanner,thread_num);
- alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
-
- hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM);
- if(hit_region_cnt<0)
- {
- DEC_SCANNER_REF(my_scanner, thread_num);
- _feather->scan_err_cnt++;
- return -1;
- }
- else if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
- {
- if(hit_region_cnt>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
- }
- _mid=grab_mid(mid, _feather, thread_num, 1);
- struct scan_region_hit_wraper region_hit_wraper;
- scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt,
- _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
- compile_ret=region_compile(_feather,_mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- }
-
- DEC_SCANNER_REF(my_scanner,thread_num);
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt,0,&start, &end,thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt,0,NULL, NULL,thread_num);
- }
- if(compile_ret==0&&hit_region_cnt>0)
- {
- return -2;
- }
- return compile_ret;
-
-}
-static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id)
-{
- memset(scan_data, 0, sizeof(scan_data_t));
- scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0);
- switch(addr->addrtype)
- {
- case ADDR_TYPE_IPV4:
- scan_data->rule_type=RULETYPE_IPv4;
- scan_data->ipv4_data.proto=proto;
- switch(child_type)
- {
- case COMPONENT_TABLE_TYPE_SOURCE_IP:
- scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
- scan_data->ipv4_data.sport=ntohs(addr->v4->source);
- break;
- case COMPONENT_TABLE_TYPE_DESTINATION_IP:
- scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr);
- scan_data->ipv4_data.sport=ntohs(addr->v4->dest);
- break;
- case COMPONENT_TABLE_TYPE_SESSION:
- scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
- scan_data->ipv4_data.sport=ntohs(addr->v4->source);
- scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr);
- scan_data->ipv4_data.dport=ntohs(addr->v4->dest);
- break;
- default:
- assert(0);
- return -1;
- }
- break;
- case ADDR_TYPE_IPV6:
- scan_data->rule_type=RULETYPE_IPv6;
- scan_data->ipv6_data.proto=proto;
- switch(child_type)
- {
- case COMPONENT_TABLE_TYPE_SOURCE_IP:
- memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
- ipv6_ntoh(scan_data->ipv6_data.saddr);
- scan_data->ipv6_data.sport=ntohs(addr->v6->source);
- break;
- case COMPONENT_TABLE_TYPE_DESTINATION_IP:
- memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr));
- ipv6_ntoh(scan_data->ipv6_data.saddr);
- scan_data->ipv6_data.sport=ntohs(addr->v6->dest);
- break;
- case COMPONENT_TABLE_TYPE_SESSION:
- memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
- ipv6_ntoh(scan_data->ipv6_data.saddr);
- scan_data->ipv6_data.sport=ntohs(addr->v6->source);
-
- memcpy(scan_data->ipv6_data.daddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.daddr));
- ipv6_ntoh(scan_data->ipv6_data.daddr);
- scan_data->ipv6_data.dport=ntohs(addr->v6->dest);
- break;
- default:
- assert(0);
- return -1;
- break;
- }
- break;
- default:
- return -1;
- }
- return 0;
-}
-static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type,
- scan_result_t *region_result, unsigned int result_num,
- int* virtual_table_id,
- rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr,
- int thread_num)
-{
- int child_table_id=0;
-
- if(child_type==COMPONENT_TABLE_TYPE_NONE)
- {
- child_table_id=parent_table_id;
- child_type=COMPONENT_TABLE_TYPE_SESSION;
- }
- else
- {
- child_table_id=Maat_table_manager_get_child_id(table_mgr, parent_table_id, child_type);
- }
- if(child_table_id<0)
- {
- return 0;
- }
-
- Maat_table_schema* real_table=Maat_table_manager_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
- if(real_table==NULL)
- {
- return 0;
- }
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(table_rt_mgr, real_table->table_id);
- if(table_rt->origin_rule_num==0)
- {
- return 0;
- }
- if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
- {
- return 0;
- }
- if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
- {
- return 0;
- }
- scan_data_t scan_data;
- int ret=ip_scan_data_set(&scan_data, addr, proto, child_type, real_table->table_id);
- if(ret<0)
- {
- return -1;
- }
- int region_ret=rulescan_search(ip_scanner, thread_num, &scan_data, region_result, result_num);
- if(region_ret>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
- }
- return region_ret;
-}
-
-int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
- ,struct ipaddr* addr,unsigned short int proto
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num)
-
-{
- int region_ret=0, compile_ret=0;
- struct _OUTER_scan_status_t* _mid=NULL;
- scan_result_t *region_result=NULL;
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_scanner* my_scanner=NULL;
- struct timespec start,end;
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- _mid=grab_mid(mid, _feather, thread_num, 0);
- scan_staus_count_inc(_mid);
- int virtual_table_id=0;
- enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID;
- table_type=Maat_table_manager_get_type_by_id(_feather->table_mgr, table_id);
- if(table_type==TABLE_TYPE_INVALID)
- {
- _feather->scan_err_cnt++;
- return -1;
- }
-
- my_scanner=_feather->scanner;
- if(my_scanner==NULL)
- {
- return 0;
- }
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, table_id);
-
- region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
- int region_hit_cnt=0;
- int region_rslt_virtual_table_id[MAX_SCANNER_HIT_NUM];
- alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
- INC_SCANNER_REF(my_scanner, thread_num);
-
- if(table_type==TABLE_TYPE_COMPOSITION)
- {
- enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION};
- for(int i=0; i<3; i++)
- {
- region_ret=IP_composition_scan(addr, proto, table_id, childs[i],
- region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id,
- my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
- if(region_ret<0)
- {
- _feather->scan_err_cnt++;
- }
- else if(region_ret>0)
- {
- for(int j=0; jregion, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
- if(region_ret<0)
- {
- _feather->scan_err_cnt++;
- }
- else if(region_ret>0)
- {
- region_hit_cnt+=region_ret;
- }
-
- }
-
- struct scan_region_hit_wraper region_hit_wraper;
- if(region_hit_cnt>0 || scan_status_should_compile_NOT(_mid) )
- {
- _mid=grab_mid(mid, _feather, thread_num, 1);
- scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_hit_cnt, -1,
- _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
- if(table_type==TABLE_TYPE_COMPOSITION)
- {
- region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id;
- }
- compile_ret=region_compile(_feather,_mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- }
- DEC_SCANNER_REF(my_scanner,thread_num);
- if(_feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num);
- }
- if(compile_ret==0&®ion_hit_cnt>0)
- {
- return -2;
- }
- return compile_ret;
-}
-int Maat_scan_addr(Maat_feather_t feather,int table_id
- ,struct ipaddr* addr
- ,struct Maat_rule_t*result,int rule_num
- ,scan_status_t *mid,int thread_num)
-{
- int compile_ret=0;
- compile_ret=Maat_scan_proto_addr(feather,table_id,
- addr, 0,
- result, rule_num,
- mid,thread_num);
- return compile_ret;
-}
-stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_scanner* scanner=NULL;
-
- struct Maat_table_schema *p_table=NULL;
- int virtual_table_id=0;
- assert(thread_num<_feather->scan_thread_num);
- p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
- if(p_table==NULL)
- {
- _feather->scan_err_cnt++;
- return NULL;
- }
-
- struct string_table_schema* expr_desc=&(p_table->expr);
- struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1);
- scanner=_feather->scanner;
- sp->feather=_feather;
- sp->p_real_table=p_table;
- sp->virtual_table_id=virtual_table_id;
- sp->last_full_version=_feather->last_full_version;
- sp->ref_scanner=_feather->scanner;
- sp->process_offset=0;
- sp->rs_stream_para=NULL;
-
- if(scanner==NULL)
- {
- return sp;
- }
-
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
- INC_SCANNER_REF(scanner, thread_num);
- alignment_int64_array_add(table_rt->stream_num, thread_num, 1);
-
- if(table_rt->origin_rule_num==0)
- {
- return sp;
- }
-
- sp->thread_num=thread_num;
- sp->max_cross_size=expr_desc->cross_cache_size;
- sp->caching_size=0;
- sp->scan_buff=NULL;
- sp->last_cache=NULL;
- if(expr_desc->do_charset_merge==1)
- {
- sp->do_merge=1;
- }
- if(table_rt->expr.expr_rule_cnt>0)
- {
- sp->do_expr=1;
- }
- if(table_rt->expr.regex_rule_cnt>0)
- {
- sp->do_regex=1;
- }
- sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
- return sp;
-}
-
-int Maat_stream_scan_string_detail(stream_para_t* stream_para
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
- ,int* detail_ret,scan_status_t* mid)
-{
- struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
- struct Maat_scanner* scanner=sp->feather->scanner;
-
- int sub_type=0;
- int region_ret=0,hit_region_cnt=0,compile_ret=0;
- int district_id=-1;
- struct _OUTER_scan_status_t* _mid=NULL;
- scan_result_t *region_result;
- scan_data_t region_scan_data;
- struct timespec start,end;
- if(sp->feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC, &start);
- }
- _mid=grab_mid(mid, sp->feather, sp->thread_num, 0);
- scan_staus_count_inc(_mid);
- if(data==NULL||data_len<=0||scanner==NULL)
- {
- return 0;
- }
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
-
-
- if(sp->last_full_version!=sp->feather->last_full_version||sp->ref_scanner!=sp->feather->scanner)
- {
- return 0;
- }
- //table rule num is already tested in Maat_stream_scan_string_start
-
-
- if(sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
- {
- sp->feather->scan_err_cnt++;
- return -1;
- }
- alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
- region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num;
- *detail_ret=0;
- if(sp->do_merge==1)
- {
- sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE, 0);
- }
- else
- {
- sub_type=make_sub_type(sp->p_real_table->table_id, charset, 0);
- }
- if(sp->max_cross_size>0&&sp->caching_size>0)
- {
- if(sp->scan_buff!=NULL)
- {
- free(sp->scan_buff);
- sp->scan_buff=NULL;
- }
- sp->scan_buff=ALLOC(char, sp->caching_size+data_len);
- memcpy(sp->scan_buff,sp->last_cache,sp->caching_size);
- memcpy(sp->scan_buff+sp->caching_size,data,data_len);
- region_scan_data.text_data.text=sp->scan_buff;
- region_scan_data.text_data.tlen=sp->caching_size+data_len;
- }
- else
- {
- region_scan_data.text_data.text=data;
- region_scan_data.text_data.tlen=data_len;
- }
- region_scan_data.text_data.toffset=(int)MIN(0xffffffff/2, sp->process_offset);//longger then int
- if(sp->last_cache==NULL&&sp->max_cross_size>0)
- {
- assert(sp->caching_size==0);
- sp->last_cache=ALLOC(char, sp->max_cross_size);
- }
- if(sp->max_cross_size>0)
- {
- sp->caching_size=detain_last_data(sp->last_cache,sp->max_cross_size,sp->caching_size,data,data_len,&(sp->process_offset));
- }
- else
- {
- sp->process_offset+=data_len;
- }
- if(sp->do_expr==1)
- {
- region_scan_data.rule_type=RULETYPE_STR;
- region_scan_data.sub_type=sub_type;
- region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result, MAX_SCANNER_HIT_NUM);
- if(region_ret<0)
- {
- sp->feather->scan_err_cnt++;
- return -1;
- }
- else if(region_ret>0)
- {
- hit_region_cnt+=region_ret;
- }
- }
- if(sp->do_regex==1)
- {
- region_scan_data.rule_type=RULETYPE_REG;
- region_scan_data.sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE,0);
- region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt);
- if(region_ret<0)
- {
- sp->feather->scan_err_cnt++;
- return -1;
- }
- else if(region_ret>0)
- {
- hit_region_cnt+=region_ret;
- }
- }
- if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS)
- {
- district_id=_mid->district_id;
- }
- if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
- {
- if(hit_region_cnt>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
- }
- _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
- struct scan_region_hit_wraper region_hit_wraper;
- scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, district_id,
- _mid->is_last_scan, sp->virtual_table_id, _mid->scan_cnt);
-
- if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid))
- {
- compile_ret=region_compile(sp->feather, _mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- sp->thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0)
- {
- if(sp->scan_buff!=NULL)
- {
- *detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->compile_mid,
- region_result, hit_region_cnt,
- result, compile_ret,
- _mid->scan_cnt,
- hit_detail, detail_num, scanner);
- }
- else
- {
- *detail_ret=fill_region_hit_detail(data, _mid->compile_mid,
- region_result, hit_region_cnt,
- result, compile_ret,
- _mid->scan_cnt,
- hit_detail, detail_num, scanner);
- }
- }
- }
- }
- if(*detail_ret==0)
- {
- free(sp->scan_buff);
- sp->scan_buff=NULL;
- }
- if(sp->feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt,data_len,&start, &end,sp->thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt,data_len,NULL, NULL,sp->thread_num);
- }
- if(compile_ret==0&&hit_region_cnt>0)
- {
- return -2;
- }
- return compile_ret;
-}
-int Maat_stream_scan_string(stream_para_t* stream_para
- ,enum MAAT_CHARSET charset,const char* data,int data_len
- ,struct Maat_rule_t*result,int* found_pos,int rule_num
- ,scan_status_t* mid)
-
-{
-
- int compile_ret=0;
- int detail_ret=0;
- compile_ret=Maat_stream_scan_string_detail(stream_para, charset,data,data_len,
- result,rule_num, NULL, 0,&detail_ret,mid);
- return compile_ret;
-}
-void Maat_stream_scan_string_end(stream_para_t* stream_para)
-{
- struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
- struct Maat_scanner* scanner=sp->feather->scanner;
- struct Maat_table_runtime* table_rt=NULL;
- if(scanner!=NULL)
- {
- table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id);
- alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
-
- }
- if(sp->rs_stream_para!=NULL)
- {
- if(scanner!=NULL&&sp->last_full_version==sp->feather->last_full_version&&sp->ref_scanner==sp->feather->scanner)
- {
-
- DEC_SCANNER_REF(scanner, sp->thread_num);
- rulescan_endstream(sp->rs_stream_para);
- }
- else
- {
- rulescan_endstream_simple(sp->rs_stream_para);
- sp->feather->zombie_rs_stream--;
- }
- sp->rs_stream_para=NULL;
- }
- if(sp->last_cache!=NULL)
- {
- free(sp->last_cache);
- sp->last_cache=NULL;
- sp->caching_size=0;
- }
- if(sp->scan_buff!=NULL)
- {
- free(sp->scan_buff);
- sp->scan_buff=NULL;
- }
- sp->ref_scanner=NULL;
- sp->feather=NULL;
- free(sp);
- *stream_para=NULL;
- return;
-}
-
-stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct Maat_scanner* scanner=NULL;
- sfh_instance_t * tmp_fuzzy_handle=NULL;
- struct Maat_table_schema *p_table=NULL;
- int virtual_table_id=0;
- p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
- if(p_table==NULL)
- {
- _feather->scan_err_cnt++;
- return NULL;
- }
- struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1);
- scanner=_feather->scanner;
- sp->feather=_feather;
- sp->ref_scanner=_feather->scanner;
- sp->p_real_table=p_table;
- sp->virtual_table_id=virtual_table_id;
- sp->last_full_version=_feather->last_full_version;
- sp->process_offset=0;
- INC_SCANNER_REF(scanner, thread_num);
- if(scanner==NULL)
- {
- return sp;
- }
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id);
- tmp_fuzzy_handle=SFH_instance(total_len);
- if(tmp_fuzzy_handle==NULL)
- {
- _feather->scan_err_cnt++;
- return NULL;
- }
-
- sp->thread_num=thread_num;
- sp->total_len=total_len;
- sp->fuzzy_hash_handle=tmp_fuzzy_handle;
- pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
- alignment_int64_array_add(table_rt->stream_num,thread_num,1);
- return sp;
-}
-
-inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size)
-{
- const unsigned long long QUERY_MIN_RATE=(3); //30%
-// const unsigned long long QUERY_MIN_LEN=(1024*1024*4);
-//do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0
- unsigned long long rate=(acc_len*10)/total_len;
-// if(acc_len>QUERY_MIN_LEN)
-// {
-// return 1;
-// }
- //TODO use a more thorough strategy.
- if(total_len==0)
- {
- return 1;
- }
- if(rate>(unsigned long long)(point_size+QUERY_MIN_RATE))
- {
- return 0;
- }
- if(rate>=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0)
- {
- query_point[rate-QUERY_MIN_RATE]=1;
- return 1;
- }
- return 0;
-}
-int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int data_len, unsigned long long offset, struct Maat_rule_t * result, int rule_num, scan_status_t * mid)
-{
- struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
- int do_query=0;
- int hit_region_cnt=0,compile_ret=0;
- struct _OUTER_scan_status_t* _mid=NULL;
-
- struct timespec start, end;
- if(sp->feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&start);
- }
- _mid=grab_mid(mid, sp->feather, sp->thread_num, 0);
- scan_staus_count_inc(_mid);
-
- if(data==NULL||data_len<=0)
- {
- return 0;
- }
- if(sp->feather->scanner==NULL)
- {
- return 0;
- }
- GIE_result_t* region_result=sp->feather->scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num;
-
- struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id);
- GIE_handle_t* GIE_handle=table_rt->similar.gie_handle;
- unsigned long long digest_len=0;
- char* digest_buff=NULL;
-
- alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
- pthread_mutex_lock(&(sp->fuzzy_mutex));
- sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
- pthread_mutex_unlock(&(sp->fuzzy_mutex));
- do_query=REACH_QUERY_THRESH(sp->total_len, sp->process_offset, sp->query_point,8);
- if(do_query==0)
- {
- goto fast_out;
- }
- pthread_mutex_lock(&(sp->fuzzy_mutex));
- digest_len=SFH_status(sp->fuzzy_hash_handle, HASH_LENGTH);
- pthread_mutex_unlock(&(sp->fuzzy_mutex));
- if(digest_len==0)
- {
- goto fast_out;
- }
- digest_buff=ALLOC(char, digest_len);
- pthread_mutex_lock(&(sp->fuzzy_mutex));
- SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
- pthread_mutex_unlock(&(sp->fuzzy_mutex));
-
- if(GIE_handle!=NULL)
- {
- hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM);
- }
-
- free(digest_buff);
- digest_buff=NULL;
- if(hit_region_cnt<0)//error occurs
- {
- sp->feather->scan_err_cnt++;
- compile_ret=-1;
- goto fast_out;
- }
- if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
- {
- if(hit_region_cnt>0)
- {
- alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
- }
- _mid=grab_mid(mid,sp->feather, sp->thread_num,1);
- struct scan_region_hit_wraper region_hit_wraper;
- scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt,
- _mid->is_last_scan, sp->virtual_table_id, _mid->scan_cnt);
- compile_ret=region_compile(sp->feather, _mid->compile_mid,
- ®ion_hit_wraper,
- result, rule_num,
- sp->thread_num);
- assert(_mid->is_last_scan<2);
- if(_mid->is_last_scan==1)
- {
- _mid->is_last_scan=2;
- }
- }
-fast_out:
- if(sp->feather->perf_on==1)
- {
- clock_gettime(CLOCK_MONOTONIC,&end);
- Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, sp->thread_num);
- }
- else
- {
- Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, sp->thread_num);
- }
- if(compile_ret==0&&hit_region_cnt>0)
- {
- return -2;
- }
- return compile_ret;
-}
-void Maat_stream_scan_digest_end(stream_para_t* stream_para)
-{
- struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
- struct Maat_scanner* scanner=sp->feather->scanner;
- struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id);
- alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
- if(scanner!=NULL)
- {
- if(sp->last_full_version==sp->feather->last_full_version)
- {
- DEC_SCANNER_REF(scanner, sp->thread_num);
- }
- }
- SFH_release(sp->fuzzy_hash_handle);
- pthread_mutex_destroy(&(sp->fuzzy_mutex));
- assert(sp->last_cache==NULL);
- assert(sp->scan_buff==NULL);
- sp->ref_scanner=NULL;
- sp->feather=NULL;
- free(sp);
- *stream_para=NULL;
-
- return;
-}
-int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size)
-{
- int ret=0;
- struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
- struct Maat_compile_rule* compile=NULL;
- switch(type)
- {
- case MAAT_RULE_SERV_DEFINE:
- compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id);
- if(compile)
- {
- ret=MIN(size, compile->head.serv_def_len);
- memcpy(value, compile->service_defined, ret);
- }
- else
- {
- ret=0;
- }
- break;
- default:
- ret=-1;
- }
- return ret;
-}
-
-int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct _OUTER_scan_status_t* _mid=NULL;
- int map_ret=-1;
- if(_feather->scanner==NULL)
- {
- return 0;
- }
- _mid=grab_mid(mid, _feather, -1, 0);
- switch(type)
- {
- case MAAT_SET_SCAN_DISTRICT:
- if(value==NULL||size<=0)
- {
- _feather->scan_err_cnt++;
- return -1;
- }
- map_ret=maat_kv_read_unNull(_feather->scanner->district_map, (const char*)value, size, &(_mid->district_id));
- if(map_ret<0)
- {
- //May be the district is not effected yet.
- _mid->district_id=DISTRICT_UNKNOWN;
- }
- _mid->is_set_district=1;
- break;
- case MAAT_SET_SCAN_LAST_REGION:
- assert(_mid->is_last_scan==0);
- _mid->is_last_scan=1;
- break;
- default:
- _feather->scan_err_cnt++;
- return -1;
- break;
- }
- return 0;
-}
-int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- struct _OUTER_scan_status_t* _mid=NULL;
- struct Maat_hit_path_t* paths;
-
- size_t n_read=0;
- int ret=0;
- _mid=grab_mid(mid, _feather, 0, 0);
-
- if(_mid->compile_mid==NULL||_feather->scanner==NULL)
- {
- return 0;
- }
- switch(type)
- {
- case MAAT_GET_SCAN_HIT_PATH:
- if(value==NULL||size<=0||size%sizeof(struct Maat_hit_path_t)!=0)
- {
- return -1;
- }
- paths=(struct Maat_hit_path_t*)value;
- n_read=size/sizeof(struct Maat_hit_path_t);
- n_read=Maat_hierarchy_get_hit_paths(_feather->scanner->hier , _mid->compile_mid, paths, n_read);
- ret=n_read;
- break;
- default:
- return -1;
- }
- return ret;
-}
-
-int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size)
-{
- struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- int * int_val=(int *)value;
- long long* longlong_val=(long long*)value;
- switch(type)
- {
- case MAAT_STATE_VERSION:
- if(size!=sizeof(long long))
- {
- return -1;
- }
- if(_feather->new_version!=-1)
- {
- *longlong_val=_feather->new_version;
- }
- else
- {
- *longlong_val=_feather->maat_version;
- }
- break;
- case MAAT_STATE_LAST_UPDATING_TABLE:
- *int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr);
- break;
- case MAAT_STATE_IN_UPDATING:
- if(size!=sizeof(int))
- {
- return -1;
- }
- if(0==pthread_mutex_trylock(&(_feather->background_update_mutex)))
- {
- *int_val=0;
- pthread_mutex_unlock(&(_feather->background_update_mutex));
- }
- else
- {
- *int_val=1;
- }
- default:
- return -1;
- break;
- }
- return 0;
-}
-
-void Maat_clean_status(scan_status_t* mid)
-{
- struct _OUTER_scan_status_t* _mid=NULL;
- if(*mid==NULL)
- {
- return;
- }
- _mid=(struct _OUTER_scan_status_t*)(*mid);
- if(_mid->thread_num>=0)
- {
- alignment_int64_array_add(_mid->feather->outer_mid_cnt, _mid->thread_num, -1);
- }
- if(_mid->compile_mid!=NULL)
- {
- Maat_hierarchy_compile_mid_free(_mid->compile_mid);
- _mid->compile_mid=NULL;
- alignment_int64_array_add(_mid->feather->compile_mid_cnt, _mid->thread_num, -1);
- }
- _mid->feather=NULL;
- free(_mid);
- *mid=NULL;
- return;
-}
-
-int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len)
-{
- return get_column_pos(line, Nth_column, column_offset, column_len);
-}
-
diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp
deleted file mode 100644
index 50ec952..0000000
--- a/src/entry/Maat_command.cpp
+++ /dev/null
@@ -1,2482 +0,0 @@
-#include "Maat_command.h"
-#include "Maat_rule.h"
-#include "Maat_rule_internal.h"
-#include "Maat_utils.h"
-#include "config_monitor.h"
-#include "map_str2int.h"
-#include "hiredis.h"
-#include
-#include
-#include
-#include
-#include
-
-#define maat_redis_monitor (module_name_str("MAAT_REDIS_MONITOR"))
-#define maat_command (module_name_str("MAAT_COMMAND"))
-const time_t MAAT_REDIS_RECONNECT_INTERVAL=5;
-const char* mr_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
-const char* mr_status_sset="MAAT_UPDATE_STATUS";
-const char* mr_expire_sset="MAAT_EXPIRE_TIMER";
-const char* mr_label_sset="MAAT_LABEL_INDEX";
-const char* mr_version_sset="MAAT_VERSION_TIMER";
-const char* mr_expire_lock="EXPIRE_OP_LOCK";
-const long mr_expire_lock_timeout=300*1000;
-const static int MAAT_REDIS_SYNC_TIME=30*60;
-const char* mr_op_str[]={"DEL","ADD","RENEW_TIMEOUT"};
-const char* foreign_source_prefix="redis://";
-const char* foreign_key_prefix="__FILE_";
-
-
-
-int _wrap_redisGetReply(redisContext *c, redisReply **reply)
-{
- return redisGetReply(c, (void **)reply);
-}
-redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...)
-{
- va_list ap;
- void *reply = NULL;
- int ret=REDIS_ERR, retry=0;
- while(reply==NULL&&retry<2&&ret!=REDIS_OK)
- {
- va_start(ap,format);
- reply = redisvCommand(c,format,ap);
- va_end(ap);
- if(reply==NULL)
- {
- ret=redisReconnect(c);
- retry++;
- }
- }
- return (redisReply *)reply;
-}
-redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger)
-{
- struct timeval connect_timeout;
- connect_timeout.tv_sec=0;
- connect_timeout.tv_usec=100*1000; // 100 ms
- redisReply* reply=NULL;
-
- redisContext * ctx;
- ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout);
- if(ctx==NULL||ctx->err)
- {
- if(logger==NULL)
- {
- printf("Unable to connect redis server %s:%d db%d, error: %s\n",
- redis_ip, redis_port, redis_db, ctx==NULL ? "Unknown" : ctx->errstr);
-
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Unable to connect redis server %s:%d db%d, error: %s",
- redis_ip, redis_port, redis_db, ctx==NULL ? "Unknown" : ctx->errstr);
- }
- if(ctx!=NULL) redisFree(ctx);
- return NULL;
- }
- redisEnableKeepAlive(ctx);
- reply=_wrap_redisCommand(ctx, "select %d",redis_db);
- freeReplyObject(reply);
- reply=NULL;
-
- return ctx;
-
-}
-
-int connect_redis_for_write(struct source_redis_ctx* mr_ctx, void* logger)
-{
- assert(mr_ctx->write_ctx==NULL);
- mr_ctx->write_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger);
- if(mr_ctx->write_ctx==NULL)
- {
- return -1;
- }
- else
- {
- return 0;
- }
-}
-redisContext* get_redis_ctx_for_write(struct _Maat_feather_t * feather)
-{
- int ret=0;
- if(feather->mr_ctx.write_ctx==NULL)
- {
- ret=connect_redis_for_write(&(feather->mr_ctx), feather->logger);
- if(ret!=0)
- {
- return NULL;
- }
- }
- return feather->mr_ctx.write_ctx;
-}
-long long read_redis_integer(const redisReply* reply)
-{
- switch(reply->type)
- {
- case REDIS_REPLY_INTEGER:
- return reply->integer;
- break;
- case REDIS_REPLY_ARRAY:
- assert(reply->element[0]->type==REDIS_REPLY_INTEGER);
- return reply->element[0]->integer;
- break;
- case REDIS_REPLY_STRING:
- return atoll(reply->str);
- break;
- default:
- return -1;
- break;
- }
- return 0;
-}
-long long redis_server_time(redisContext* ctx)
-{
- long long server_time=0;
- redisReply* data_reply=NULL;
- data_reply=_wrap_redisCommand(ctx,"TIME");
- if(data_reply->type==REDIS_REPLY_ARRAY)
- {
- server_time=atoll(data_reply->element[0]->str);
- freeReplyObject(data_reply);
- data_reply=NULL;
- }
- return server_time;
-}
-enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
-{
- enum MAAT_TABLE_TYPE ret=TABLE_TYPE_IP;
- switch(p->region_type)
- {
- case REGION_IP:
- ret=TABLE_TYPE_IP;
- break;
- case REGION_EXPR:
- if(p->expr_rule.district==NULL)
- {
- ret=TABLE_TYPE_EXPR;
- }
- else
- {
- ret=TABLE_TYPE_EXPR_PLUS;
- }
- break;
- case REGION_INTERVAL:
- if(p->interval_rule.district==NULL)
- {
- ret=TABLE_TYPE_INTERVAL;
- }
- else
- {
- ret=TABLE_TYPE_INTERVAL_PLUS;
- }
- break;
- case REGION_DIGEST:
- ret=TABLE_TYPE_DIGEST;
- break;
- case REGION_SIMILARITY:
- ret=TABLE_TYPE_SIMILARITY;
- break;
- default:
- assert(0);
- }
- return ret;
-}
-int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type, int valid_column_seq)
-{
- size_t offset=0, len=0;
- unsigned int column_seq=0, ret=0;
- switch(type)
- {
- case TABLE_TYPE_EXPR:
- column_seq=7;
- break;
- case TABLE_TYPE_IP:
- column_seq=14;
- break;
- case TABLE_TYPE_IP_PLUS:
- column_seq=18;
- break;
- case TABLE_TYPE_COMPILE:
- column_seq=8;
- break;
- case TABLE_TYPE_PLUGIN:
- case TABLE_TYPE_IP_PLUGIN:
- case TABLE_TYPE_FQDN_PLUGIN:
- case TABLE_TYPE_BOOL_PLUGIN:
- if(valid_column_seq<0)
- {
- return -1;
- }
- column_seq=(unsigned int)valid_column_seq;
- break;
- case TABLE_TYPE_INTERVAL:
- column_seq=5;
- break;
- case TABLE_TYPE_INTERVAL_PLUS:
- column_seq=6;
- break;
- case TABLE_TYPE_DIGEST:
- column_seq=6;
- break;
- case TABLE_TYPE_SIMILARITY:
- column_seq=5;
- break;
- case TABLE_TYPE_EXPR_PLUS:
- column_seq=8;
- break;
- case TABLE_TYPE_GROUP2COMPILE:
- case TABLE_TYPE_GROUP2GROUP:
- column_seq=3;
- break;
- default:
- assert(0);
- }
-
- ret=get_column_pos(line, column_seq, &offset, &len);
- if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer.
- {
- return -1;
- }
- return offset;
-}
-int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
-{
- int i=0;
- i=get_valid_flag_offset(line, type,valid_column_seq);
- if(i<0)
- {
- return -1;
- }
- line[i]='0';
- return 0;
-}
-
-void serialize_group2group(enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g, char* buff, size_t sz)
-{
- snprintf(buff, sz, "%d\t%d\t%d", g2g->group_id,
- g2g->superior_group_id,
- op);
- return;
-}
-void serialize_group2compile(enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c, char* buff, size_t sz)
-{
- snprintf(buff, sz, "%d\t%d\t%d\t%d\t%s\t%d", g2c->group_id,
- g2c->compile_id,
- op,
- g2c->not_flag,
- g2c->virtual_table_name?g2c->virtual_table_name:"null",
- g2c->clause_index);
- return;
-}
-void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int clause_num, enum MAAT_OPERATION op, char* buff, size_t sz)
-{
- if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
- const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null");
-
- snprintf(buff, sz, "%d\t%d\t%hhu\t%hhu\t%hhu\t0\t%s\t%d\t%d",
- p_m_rule->config_id,
- p_m_rule->service_id,
- p_m_rule->action,
- p_m_rule->do_blacklist,
- p_m_rule->do_log,
- service_define,
- op,
- clause_num);
- return;
-}
-void serialize_region(const struct Maat_cmd_region* p, int group_id, char* buff, size_t sz)
-{
- UNUSED size_t ret=0;
- switch(p->region_type)
- {
- case REGION_IP:
- ret=snprintf(buff, sz, "%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1",
- p->region_id,
- group_id,
- p->ip_rule.addr_type,
- p->ip_rule.src_ip,
- p->ip_rule.mask_src_ip,
- p->ip_rule.src_port,
- p->ip_rule.mask_src_port,
- p->ip_rule.dst_ip,
- p->ip_rule.mask_dst_ip,
- p->ip_rule.dst_port,
- p->ip_rule.mask_dst_port,
- p->ip_rule.protocol,
- p->ip_rule.direction);
- break;
- case REGION_IP_PLUS:
- ret=snprintf(buff, sz, "%d\t%d\t%d\t%s\t%s\t%s\t%s\t%hu\t%hu\t%s\t%s\t%s\t%s\t%hu\t%hu\t%d\t%d\t1",
- p->region_id,
- group_id,
- p->ip_plus_rule.addr_type,
- p->ip_plus_rule.saddr_format,
- p->ip_plus_rule.src_ip1,
- p->ip_plus_rule.src_ip2,
- p->ip_plus_rule.sport_format,
- p->ip_plus_rule.src_port1,
- p->ip_plus_rule.src_port2,
- p->ip_plus_rule.daddr_format,
- p->ip_plus_rule.dst_ip1,
- p->ip_plus_rule.dst_ip2,
- p->ip_plus_rule.dport_format,
- p->ip_plus_rule.dst_port1,
- p->ip_plus_rule.dst_port2,
- p->ip_plus_rule.protocol,
- p->ip_plus_rule.direction);
- break;
- case REGION_EXPR:
- if(p->expr_rule.district==NULL)
- {
- ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1",
- p->region_id,
- group_id,
- p->expr_rule.keywords,
- p->expr_rule.expr_type,
- p->expr_rule.match_method,
- p->expr_rule.hex_bin);
- }
- else //expr_plus
- {
- ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1",
- p->region_id,
- group_id,
- p->expr_rule.district,
- p->expr_rule.keywords,
- p->expr_rule.expr_type,
- p->expr_rule.match_method,
- p->expr_rule.hex_bin);
- }
- break;
- case REGION_INTERVAL:
- ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1",
- p->region_id,
- group_id,
- p->interval_rule.low_boundary,
- p->interval_rule.up_boundary);
- break;
- case REGION_DIGEST:
- ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1",
- p->region_id,
- group_id,
- p->digest_rule.orgin_len,
- p->digest_rule.digest_string,
- p->digest_rule.confidence_degree);
- break;
- case REGION_SIMILARITY:
- ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1",
- p->region_id,
- group_id,
- p->similarity_rule.target,
- p->similarity_rule.threshold);
- break;
- default:
- assert(0);
- }
- assert(rettable_line!=NULL)
- {
- free(rule->table_line);
- }
- if(rule->n_foreign>0)
- {
- for(int i=0; in_foreign; i++)
- {
- free(rule->f_keys[i].filename);
- free(rule->f_keys[i].key);
- }
- free(rule->f_keys);
- }
- memset(rule,0,sizeof(struct serial_rule_t));
- return;
-}
-void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op, unsigned long rule_id,int label_id,const char* table_name,const char* line, long long timeout)
-{
- memset(rule, 0, sizeof(struct serial_rule_t));
- rule->op=op;
- rule->rule_id=rule_id;
- rule->label_id=label_id;
- rule->timeout=timeout;
- assert(strlen(table_name)table_name));
- strncpy(rule->table_name, table_name, sizeof(rule->table_name));
- if(line!=NULL)
- {
- rule->table_line=_maat_strdup(line);
- }
- return;
-}
-int get_inc_key_list(long long instance_version, long long target_version, redisContext *c, struct serial_rule_t** list,void* logger)
-{
- redisReply* reply=NULL,*tmp_reply=NULL;
- char op_str[4];
- int rule_num=0;
- UNUSED int ret=0;
- unsigned int i=0, j=0;
- long long nearest_rule_version;
- struct serial_rule_t *s_rule=NULL;
-
- //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version.
- //The elements are considered to be ordered from low to high scores(instance_version).
- reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",mr_status_sset,instance_version,target_version);
-
- if(reply==NULL)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "GET %s failed with a NULL reply, error: %s.", mr_status_sset, c->errstr);
- return -1;
- }
- assert(reply->type==REDIS_REPLY_ARRAY);
- rule_num=reply->elements;
- if(reply->elements==0)
- {
- freeReplyObject(reply);
- reply=NULL;
- return 0;
- }
-
- tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s", mr_status_sset,reply->element[0]->str);
- if(tmp_reply->type!=REDIS_REPLY_STRING)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
- "ZSCORE %s %s failed Version: %lld->%lld", mr_status_sset, reply->element[0]->str, instance_version, target_version);
- freeReplyObject(tmp_reply);
- tmp_reply=NULL;
- freeReplyObject(reply);
- reply=NULL;
- return -1;
- }
- nearest_rule_version=read_redis_integer(tmp_reply);
- freeReplyObject(tmp_reply);
- tmp_reply=NULL;
- if(nearest_rule_version<0)
- {
- return -1;
- }
- if(nearest_rule_version!=instance_version+1)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
- "Noncontinuous VERSION Redis: %lld MAAT: %lld.", nearest_rule_version, instance_version);
- }
- s_rule=ALLOC(struct serial_rule_t, reply->elements);
- for(i=0, j=0;ielements;i++)
- {
- assert(reply->element[i]->type==REDIS_REPLY_STRING);
- ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%lu", op_str, s_rule[j].table_name, &(s_rule[j].rule_id));
- if(ret!=3||s_rule[i].rule_id<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key: %s", reply->element[i]->str);
- continue;
- }
- if(strncmp(op_str,"ADD",strlen("ADD"))==0)
- {
- s_rule[j].op=MAAT_OP_ADD;
- }
- else if(strncmp(op_str,"DEL",strlen("DEL"))==0)
- {
- s_rule[j].op=MAAT_OP_DEL;
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key: %s", reply->element[i]->str);
- continue;
- }
- j++;
- }
- rule_num=j;
- *list=s_rule;
- freeReplyObject(reply);
- reply=NULL;
-
- return rule_num;
-}
-struct s_rule_array_t
-{
- int cnt;
- int size;
- struct serial_rule_t* array;
-};
-void save_serial_rule_cb(const uchar * key, uint size, void * data, void * user)
-{
- struct s_rule_array_t* array=(struct s_rule_array_t*)user;
- int i=array->cnt;
- memcpy(&(array->array[i]),data,sizeof(struct serial_rule_t));
- array->array[i].op=MAAT_OP_ADD;
- return;
-}
-int recovery_history_version(const struct serial_rule_t* current, int current_num, const struct serial_rule_t* changed, int changed_num, struct serial_rule_t** history_result)
-{
- int i=0,ret=0;
- unsigned int history_num=0;
- int hash_slot_size=1;
- MESA_htable_handle htable=NULL;
- MESA_htable_create_args_t hargs;
- struct s_rule_array_t tmp_array;
- char hkey[256+20];
- int tmp=current_num+changed_num;
- for(;tmp>0;tmp=tmp/2)
- {
- hash_slot_size*=2;
- }
-
- memset(&hargs,0,sizeof(hargs));
- hargs.thread_safe=0;
- hargs.hash_slot_size = hash_slot_size;
- hargs.max_elem_num = 0;
- hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
- hargs.expire_time = 0;
- hargs.key_comp = NULL;
- hargs.key2index = NULL;
- hargs.recursive = 1;
- hargs.data_free = NULL;//data is an reference, no need to free.
- hargs.data_expire_with_condition = NULL;
- htable=MESA_htable_create(&hargs, sizeof(hargs));
- MESA_htable_print_crtl(htable, 0);
-
- for(i=0;i0);
- }
-
- for(i=changed_num-1;i>=0;i--)
- {
- snprintf(hkey,sizeof(hkey),"%ld,%s",changed[i].rule_id,changed[i].table_name);
- if(changed[i].op==MAAT_OP_ADD)//newly added rule is need to delete from current, so that history version can be recovered.
- {
- ret=MESA_htable_del(htable, (uchar*)hkey, strlen(hkey), NULL);
- }
- else
- {
- ret=MESA_htable_add(htable, (uchar*)hkey, strlen(hkey),changed+i);
- }
- if(ret<0)//failed
- {
- goto error_out;
- }
- }
- history_num=MESA_htable_get_elem_num(htable);
- tmp_array.cnt=0;
- tmp_array.size=history_num;
- tmp_array.array=(struct serial_rule_t*)calloc(history_num,sizeof(struct serial_rule_t));
- MESA_htable_iterate(htable, save_serial_rule_cb, &tmp_array);
- *history_result=tmp_array.array;
- ret=history_num;
-error_out:
- MESA_htable_destroy(htable, NULL);
- return ret;
-}
-
-int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct Maat_table_manager* table_mgr, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off)
-{
- redisReply* reply=NULL,*sub_reply=NULL;
- long long redis_version=0,target_version=0;
- int rule_num=0, changed_rule_num=0, table_id=0;
- int ret=0;
- unsigned int i=0,full_idx =0,append_cmd_cnt=0;
- struct serial_rule_t *s_rule_array=NULL, *changed_rule_array=NULL, *history_rule_array=NULL;
-
- reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION");
- if(reply!=NULL)
- {
-
- if(reply->type==REDIS_REPLY_NIL||reply->type==REDIS_REPLY_ERROR)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, "GET MAAT_VERSION failed, maybe Redis is busy.");
- freeReplyObject(reply);
- reply=NULL;
- return -1;
- }
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "GET MAAT_VERSION failed with NULL reply, error: %s.", c->errstr);
- return -1;
- }
- redis_version=read_redis_integer(reply);
- if(redis_version<0)
- {
- if(reply->type==REDIS_REPLY_ERROR)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Redis Communication error: %s.", reply->str);
- }
- return -1;
- }
- freeReplyObject(reply);
- reply=NULL;
- if(redis_version==instance_version)
- {
- return 0;
- }
-
- if(instance_version==0||desired_version!=0)
- {
- goto FULL_UPDATE;
- }
- if(redis_version Redis: %lld.", instance_version, redis_version);
- goto FULL_UPDATE;
- }
- if(redis_version>instance_version&&cumulative_off==1)
- {
- target_version=instance_version;
- }
- else
- {
- target_version=redis_version-1;
- }
- do{
- target_version++;
- rule_num=get_inc_key_list(instance_version, target_version, c, &s_rule_array,logger);
- if(rule_num>0)
- {
- break;
- }
- else if(rule_num<0)
- {
- goto FULL_UPDATE;
- }
- else
- {
- //ret=0, nothing to do.
- }
-
- }while(rule_num==0&&target_version<=redis_version&&cumulative_off==1);
- if(rule_num==0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_redis_monitor, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s",
- mr_status_sset, instance_version, target_version-1, cumulative_off==1?"OFF":"ON");
- return 0;
- }
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
- "Inc Update from instance_version %lld to %lld (%d entries).", instance_version,target_version,rule_num);
- *list=s_rule_array;
- *update_type=CM_UPDATE_TYPE_INC;
- *new_version=target_version;
- return rule_num;
-
-FULL_UPDATE:
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Initiate full udpate from instance_version %d to %lld.", instance_version, desired_version==0?redis_version:desired_version);
- append_cmd_cnt=0;
- ret=redisAppendCommand(c, "MULTI");
- append_cmd_cnt++;
- ret=redisAppendCommand(c, "GET MAAT_VERSION");
- append_cmd_cnt++;
- ret=redisAppendCommand(c, "KEYS EFFECTIVE_RULE:*");
- append_cmd_cnt++;
- //consume reply "OK" and "QUEUED".
- for(i=0;ierrstr);
- return -1;
- }
- if(reply->type!=REDIS_REPLY_ARRAY)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key List type %d", reply->type);
- freeReplyObject(reply);
- reply=NULL;
- return -1;
- }
- *new_version=read_redis_integer(reply->element[0]);
- sub_reply=reply->element[1];
- if(sub_reply->type!=REDIS_REPLY_ARRAY)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key List type %d", sub_reply->type);
- freeReplyObject(reply);
- reply=NULL;
- return -1;
- }
-
- s_rule_array=(struct serial_rule_t*)calloc(sub_reply->elements,sizeof(struct serial_rule_t));
- for(i=0, full_idx=0; ielements; i++)
- {
- if(sub_reply->element[i]->type!=REDIS_REPLY_STRING)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key Type: %d", sub_reply->element[i]->type);
- continue;
- }
- ret=sscanf(sub_reply->element[i]->str,"%*[^:]:%[^,],%ld",s_rule_array[full_idx].table_name,&(s_rule_array[full_idx].rule_id));
- s_rule_array[full_idx].op=MAAT_OP_ADD;
- if(ret!=2||s_rule_array[full_idx].rule_id<0||strlen(s_rule_array[full_idx].table_name)==0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Invalid Redis Key Format: %s", sub_reply->element[i]->str);
- continue;
- }
- if(table_mgr)
- {
- table_id=Maat_table_manager_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name);
- if(table_id<0)//Unrecognized table.
- {
- continue;
- }
- }
- full_idx++;
- }
- rule_num=full_idx;
- freeReplyObject(reply);
- reply=NULL;
- if(desired_version!=0)
- {
- changed_rule_num=get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger);
- if(changed_rule_num<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Recover history version %lld faild where as redis version is %lld.", desired_version, redis_version);
- }
- else if(changed_rule_num==0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Nothing to recover from history version %lld to redis version is %lld.", desired_version, redis_version);
- }
- else
- {
- ret=recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array);
- if(ret>0)
- {
- free(s_rule_array);
- s_rule_array=history_rule_array;
- rule_num=ret;
- *new_version=desired_version;
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Successfully recovered from history version %lld to redis version is %lld.", desired_version, redis_version);
- }
- }
- free(changed_rule_array);
- }
- *list=s_rule_array;
- *update_type=CM_UPDATE_TYPE_FULL;
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
- "Full update %d keys of version %lld.", rule_num, *new_version);
-
- return rule_num ;
-}
-
-int _get_maat_redis_value(redisContext *c, struct serial_rule_t* rule_list, int rule_num, void* logger)
-{
- int i=0,failed_cnt=0,idx=0;
- UNUSED int ret=0;
- int error_happened=0;
- int *retry_ids=(int*)malloc(sizeof(int)*rule_num);
- char redis_cmd[256];
- redisReply* reply=NULL;
- for(i=0;itype==REDIS_REPLY_STRING)
- {
- rule_list[i].table_line=_maat_strdup(reply->str);
- }
- else
- {
- if(reply->type==REDIS_REPLY_NIL)
- {
- retry_ids[failed_cnt]=i;
- failed_cnt++;
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Redis GET %s:%s,%d failed",mr_key_prefix[rule_list[i].op],
- rule_list[i].table_name,
- rule_list[i].rule_id);
- error_happened=1;
- }
- }
- freeReplyObject(reply);
- reply=NULL;
- }
- if(error_happened==1)
- {
- free(retry_ids);
- return -1;
- }
-
- for(i=0;itype==REDIS_REPLY_STRING)
- {
- rule_list[idx].table_line=_maat_strdup(reply->str);
- }
- else if(reply->type==REDIS_REPLY_ERROR)//Deal with Redis response: "Loading Redis is loading the database in memory"
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "redis command %s error, reply type=%d, error str=%s", redis_cmd, reply->type, reply->str);
- }
- else //Handle type "nil"
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "redis command %s failed, reply type=%d", redis_cmd, reply->type);
- }
-
- freeReplyObject(reply);
- reply=NULL;
-
- }
- free(retry_ids);
- return 0;
-}
-int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process)
-{
- int max_redis_batch=4*1024,batch_cnt=0;
- int success_cnt=0,ret=0;
- int next_print=10;
- while(success_cntnext_print)
- {
- printf(" >%d%%",next_print);
- next_print+=10;
- }
- }
- }
- if(print_process==1)
- {
- printf(" >100%%\n");
- }
- return 0;
-}
-
-int mr_transaction_success(redisReply* data_reply)
-{
- if(data_reply->type==REDIS_REPLY_NIL)
- {
- return 0;
- }
- else
- {
- return 1;
- }
-}
-
-int redlock_try_lock(redisContext *ctx, const char* lock_name, long long expire)
-{
- redisReply* reply=NULL;
- int ret=0;
- reply=_wrap_redisCommand(ctx,"SET %s locked NX PX %lld", lock_name, expire);
- if(reply->type==REDIS_REPLY_NIL)
- {
- ret=0;
- }
- else
- {
- ret=1;
- }
- freeReplyObject(reply);
- reply=NULL;
-
- return ret;
-}
-void redlock_unlock(redisContext * ctx, const char * lock_name)
-{
- redisReply* reply=NULL;
- reply=_wrap_redisCommand(ctx,"DEL %s", lock_name);
- freeReplyObject(reply);
- reply=NULL;
-
-}
-#define POSSIBLE_REDIS_REPLY_SIZE 2
-struct expected_reply
-{
- int srule_seq;
- int possible_reply_num;
- redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
-};
-void expected_reply_add(struct expected_reply* expected, int srule_seq, int type, long long integer)
-{
- int i=expected->possible_reply_num;
- assert(isrule_seq=srule_seq;
- expected->possible_replies[i].type=type;
- expected->possible_replies[i].integer=integer;
- expected->possible_reply_num++;
-}
-int mr_operation_success(redisReply* actual_reply, struct expected_reply* expected)
-{
- int i=0;
- if(expected->possible_replies[0].type!=actual_reply->type)
- {
- return 0;
- }
- for(i=0; i< expected->possible_reply_num; i++)
- {
- if(expected->possible_replies[i].type==REDIS_REPLY_INTEGER &&
- expected->possible_replies[i].type==actual_reply->type &&
- expected->possible_replies[i].integer==actual_reply->integer)
- {
- return 1;
- }
- if(expected->possible_replies[i].type==REDIS_REPLY_STATUS &&
- expected->possible_replies[i].type==actual_reply->type &&
- 0==strcasecmp(actual_reply->str, "OK"))
- {
- return 1;
- }
- }
- return 0;
-
-}
-
-long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule_num,int *renew_allowed, long long *transaction_version)
-{
- int ret=-1;
- redisReply* data_reply=NULL;
- if(renew_rule_num>0)
- {
- while(0==redlock_try_lock(ctx, mr_expire_lock, mr_expire_lock_timeout))
- {
- usleep(1000);
- }
- *renew_allowed=1;
- }
- if(rule_num>renew_rule_num)
- {
- data_reply=_wrap_redisCommand(ctx, "INCRBY MAAT_PRE_VER 1");
- *transaction_version=read_redis_integer(data_reply);
- freeReplyObject(data_reply);
- data_reply=NULL;
- if(*transaction_version<0)
- {
- return -1;
- }
- }
- if(*renew_allowed==1||rule_num>renew_rule_num)
- {
- data_reply=_wrap_redisCommand(ctx,"MULTI");
- freeReplyObject(data_reply);
- data_reply=NULL;
- ret=0;
- }
- return ret;
-}
-//parameters: 4 keys: MAAT_VERSION MAAT_UPDATE_STATUS MAAT_VERSION_TIMER MAAT_TRANSACTION_xx, 1 args: SERVER_TIME
-const char* lua_exec_done=
-"local maat_version=redis.call(\'incrby\', KEYS[1], 1);"
-"local transaction=redis.call(\'lrange\', KEYS[4], 0, -1);"
-"for k,v in pairs(transaction) do"
-" redis.call(\'zadd\', KEYS[2], maat_version, v);"
-"end;"
-"redis.call(\'del\', KEYS[4]);"
-"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);"
-"return maat_version;";
-redisReply* _exec_serial_rule_end(redisContext* ctx, const char* transaction_list, long long server_time, int renew_allowed, struct expected_reply* expect_reply, unsigned int *cnt)
-{
- redisReply* data_reply=NULL;
- if(renew_allowed==1)
- {
- redlock_unlock(ctx, mr_expire_lock);
- expect_reply[*cnt].srule_seq=-1;
- (*cnt)++;
- }
- if(strlen(transaction_list)>0)
- {
- data_reply=_wrap_redisCommand(ctx, "eval %s 4 MAAT_VERSION %s %s %s %lld",
- lua_exec_done,
- mr_status_sset,
- mr_version_sset,
- transaction_list,
- server_time);
- freeReplyObject(data_reply);
- data_reply=NULL;
- expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- }
- data_reply=_wrap_redisCommand(ctx,"EXEC");
- return data_reply;
-}
-
-void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply* expect_reply, unsigned int *cnt, int offset,int renew_allowed)
-{
- redisReply* data_reply=NULL;
- unsigned int append_cmd_cnt=0, i=0;
- for(i=0;i0)
- {
- redisAppendCommand(ctx,"ZADD %s %lld %s,%lu",
- mr_expire_sset,
- s_rule[i].timeout,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- append_cmd_cnt++;
- }
- if(s_rule[i].label_id>0)
- {
- redisAppendCommand(ctx,"ZADD %s %d %s,%lu",
- mr_label_sset,
- s_rule[i].label_id,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
-
- (*cnt)++;
-
- append_cmd_cnt++;
- }
- break;
- case MAAT_OP_DEL:
- redisAppendCommand(ctx,"RENAME %s:%s,%lu %s:%s,%lu",
- mr_key_prefix[MAAT_OP_ADD],
- s_rule[i].table_name,
- s_rule[i].rule_id,
- mr_key_prefix[MAAT_OP_DEL],
- s_rule[i].table_name,
- s_rule[i].rule_id
- );
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0);
- (*cnt)++;
- append_cmd_cnt++;
-
- redisAppendCommand(ctx,"EXPIRE %s:%s,%lu %d",
- mr_key_prefix[MAAT_OP_DEL],
- s_rule[i].table_name,
- s_rule[i].rule_id,
- MAAT_REDIS_SYNC_TIME);
- expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
- (*cnt)++;
- append_cmd_cnt++;
-
- //NX: Don't update already exisiting elements. Always add new elements.
- redisAppendCommand(ctx,"RPUSH %s DEL,%s,%lu",
- transaction_list,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- append_cmd_cnt++;
-
- // Try to remove from expiration sorted set, no matter wheather it exists or not.
- redisAppendCommand(ctx,"ZREM %s %s,%lu",
- mr_expire_sset,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- append_cmd_cnt++;
-
- // Try to remove from label sorted set, no matter wheather it exists or not.
- redisAppendCommand(ctx,"ZREM %s %s,%lu",
- mr_label_sset,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- append_cmd_cnt++;
- break;
- case MAAT_OP_RENEW_TIMEOUT:
- if(renew_allowed!=1)
- {
- continue;
- }
- //s_rule[i].timeout>0 was checked by caller.
- redisAppendCommand(ctx,"ZADD %s %lld %s,%lu",
- mr_expire_sset,
- s_rule[i].timeout,
- s_rule[i].table_name,
- s_rule[i].rule_id);
- expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
- (*cnt)++;
- append_cmd_cnt++;
-
- break;
- default:
- assert(0);
- break;
- }
- }
- for(i=0;i0)
- {
- snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version);
- }
- while(success_cntelements==multi_cmd_cnt);
- for(i=0;ielement[i];
- //failed is acceptable
- //or transaciton is success
- //or continuation of last failed
- if(expected_reply[i].srule_seq==-1||1==mr_operation_success(p, expected_reply+i)||last_failed==expected_reply[i].srule_seq)
- {
- continue;
- }
- rule_seq=expected_reply[i].srule_seq;
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_command,
- "%s %s %d failed, rule id maybe conflict or not exist.",
- mr_op_str[s_rule[rule_seq].op],
- s_rule[rule_seq].table_name,s_rule[rule_seq].rule_id);
- success_cnt--;
- last_failed=rule_seq;
- }
- }
- else
- {
- success_cnt=-1;
- }
- if(transaction_version>0)
- {
- transaction_finished_version=read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]);
- MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_command,
- "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld ",
- transaction_version,
- transaction_finished_version);
- }
-
- freeReplyObject(transaction_reply);
- transaction_reply=NULL;
-
-error_out:
- if(renew_num>0&&renew_allowed!=1)
- {
- for(i=0;i<(unsigned int)serial_rule_num;i++)
- {
- if(s_rule[i].op==MAAT_OP_RENEW_TIMEOUT)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_command
- ,"%s %s %d is not allowed due to lock contention.",mr_op_str[MAAT_OP_RENEW_TIMEOUT]
- , s_rule[i].table_name,s_rule[i].rule_id);
- }
- }
- if(success_cnt>0)
- {
- success_cnt-=renew_num;
- }
- }
- free(expected_reply);
- return success_cnt;
-}
-
-
-void check_maat_expiration(redisContext *ctx, void *logger)
-{
- unsigned int i=0,s_rule_num=0;
- UNUSED int ret=0;
- int success_cnt=0;
- redisReply* data_reply=NULL;
- struct serial_rule_t* s_rule=NULL;
- long long server_time=0;
-
- server_time=redis_server_time(ctx);
- if(!server_time)
- {
- return;
- }
- data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time);
- if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0)
- {
- freeReplyObject(data_reply);
- data_reply=NULL;
- return;
- }
- s_rule_num=data_reply->elements;
- s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),s_rule_num);
- for(i=0;ielement[i]->str,"%[^,],%ld",s_rule[i].table_name,&(s_rule[i].rule_id));
- assert(ret==2);
- }
- freeReplyObject(data_reply);
- data_reply=NULL;
- success_cnt=exec_serial_rule(ctx,s_rule, s_rule_num,server_time, logger);
-
- if(success_cnt==(int)s_rule_num)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
- ,"Succesfully expired %d rules in Redis.", s_rule_num);
- }
- else
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
- ,"Failed to expired %d of %d rules in Redis, try later.", s_rule_num-success_cnt,s_rule_num);
- }
-
- free(s_rule);
- return;
-}
-void cleanup_update_status(redisContext *ctx, void *logger)
-{
- redisReply* reply=NULL,*sub_reply=NULL;
- int append_cmd_cnt=0,i=0;
- long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0;
-
- server_time=redis_server_time(ctx);
- if(!server_time)
- {
- return;
- }
- reply=_wrap_redisCommand(ctx,"MULTI");
- freeReplyObject(reply);
- reply=NULL;
- redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
- append_cmd_cnt++;
- redisAppendCommand(ctx, "ZREMRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
- append_cmd_cnt++;
- //consume reply "OK" and "QUEUED".
- for(i=0;itype!=REDIS_REPLY_ARRAY)
- {
- goto error_out;
- }
- sub_reply=reply->element[0];
- if(sub_reply->type!=REDIS_REPLY_ARRAY)
- {
- goto error_out;
- }
- version_num=sub_reply->elements;
- if(version_num==0)
- {
- goto error_out;
- }
- version_lower_bound=read_redis_integer(sub_reply->element[0]);
- version_upper_bound=read_redis_integer(sub_reply->element[sub_reply->elements-1]);
- freeReplyObject(reply);
- reply=NULL;
-
- //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
- reply=_wrap_redisCommand(ctx,"ZREMRANGEBYSCORE %s %lld %lld",mr_status_sset,version_lower_bound,version_upper_bound);
- entry_num=read_redis_integer(reply);
- freeReplyObject(reply);
- reply=NULL;
-
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
- ,"Clean up update status from version %lld to %lld (%lld versions, %lld entries)."
- ,version_lower_bound
- ,version_upper_bound
- ,version_num
- ,entry_num);
- return;
-
-error_out:
- freeReplyObject(reply);
- reply=NULL;
- return;
-}
-const char* find_Nth_column(const char* line, int Nth, int* column_len)
-{
- size_t i=0;
- int j=0;
- int start=0, end=0;
- size_t line_len= strlen(line);
- for(i=0;in_foreign; i++)
- {
- fn_size+=strlen(p->f_keys[i].filename);
- }
-
- rewrite_line=(char*)calloc(sizeof(char), strlen(p->table_line)+fn_size);
- pos_origin_line=p->table_line;
- pos_rewrite_line=rewrite_line;
-
- for(i=0; in_foreign; i++)
- {
- origin_column=find_Nth_column(p->table_line, p->f_keys[i].column, &origin_column_size);
- strncat(pos_rewrite_line, pos_origin_line, origin_column-pos_origin_line);
- pos_rewrite_line+=origin_column-pos_origin_line;
- pos_origin_line=origin_column+origin_column_size;
-
- strncat(pos_rewrite_line, p->f_keys[i].filename, strlen(p->f_keys[i].filename));
- pos_rewrite_line+=strlen(p->f_keys[i].filename);
- }
- strncat(pos_rewrite_line, pos_origin_line, strlen(p->table_line)-(pos_origin_line-p->table_line));
-
- free(p->table_line);
- p->table_line=rewrite_line;
- return;
-}
-void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger)
-{
- int i=0;
- const char* p_foreign=NULL;
- int foreign_key_size=0;
- p_rule->f_keys=ALLOC(struct foreign_key, n_foreign);
- for(i=0; itable_line, foreign_columns[i], &foreign_key_size);
- if(p_foreign==NULL)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Get %s,%d foreign keys failed: No %dth column.",
- p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
- continue;
- }
- if(0==strncasecmp(p_foreign, "null", strlen("null")))
- {//emtpy file
- continue;
- }
- if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Get %s,%d foreign key failed: Invalid source prefix %s.",
- p_rule->table_name, p_rule->rule_id, p_foreign);
- continue;
- }
- p_rule->f_keys[p_rule->n_foreign].column=foreign_columns[i];
- foreign_key_size=foreign_key_size-strlen(foreign_source_prefix);
- p_foreign+=strlen(foreign_source_prefix);
- if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix)))
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
- "%s, %d foreign key prefix %s is not recommended.",
- p_rule->table_name, p_rule->rule_id, p_foreign);
- }
- p_rule->f_keys[p_rule->n_foreign].key=ALLOC(char, foreign_key_size+1);
- memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size);
- p_rule->f_keys[p_rule->n_foreign].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir);
- p_rule->n_foreign++;
- }
- if(p_rule->n_foreign==0)
- {
- free(p_rule->f_keys);
- p_rule->f_keys=NULL;
- }
- return;
-}
-int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
-{
- int i=0;
- int rule_with_foreign_key=0;
- struct Maat_table_schema* p_table=NULL;
- struct plugin_table_schema* plugin_desc=NULL;
- for(i=0; itable_mgr, rule_list[i].table_name);
- if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN)
- {
- continue;
- }
- plugin_desc= &(p_table->plugin);
- if(plugin_desc->n_foreign==0)
- {
- continue;
- }
- _get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger);
- rule_with_foreign_key++;
- }
- return rule_with_foreign_key;
-}
-int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger)
-{
- int i=0, j=0, foreign_key_size=0;
- int rule_with_foreign_key=0;
- const char* p_foreign=NULL;
-
- int n_foreign=0;
- int foreign_columns[MAX_FOREIGN_CLMN_NUM];
- for(i=0; i(int)strlen(foreign_source_prefix)&&0==strncmp(p_foreign,foreign_source_prefix, strlen(foreign_source_prefix)))
- {
- foreign_columns[n_foreign]=j;
- n_foreign++;
- }
- j++;
- }while(p_foreign!=NULL&&n_foreign0)
- {
- _get_foregin_keys(rule_list+i, foreign_columns, n_foreign,dir,logger);
- rule_with_foreign_key++;
- }
- }
- return rule_with_foreign_key;
-}
-
-struct foreign_conts_track
-{
- int rule_idx;
- int foreign_idx;
-};
-void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger)
-{
- int i=0, j=0;
- UNUSED int ret=0;
- int key_num=0;
- struct foreign_conts_track* track=ALLOC(struct foreign_conts_track, rule_num*MAX_FOREIGN_CLMN_NUM);
- char redis_cmd[256];
- redisReply* reply=NULL;
- struct serial_rule_t*p=NULL;
- FILE* fp=NULL;
- struct stat file_info;
-
- for(i=0;in_foreign==0)
- {
- continue;
- }
- if(p->op==MAAT_OP_DEL)
- {
- for(j=0; jn_foreign; j++)
- {
- if(rule_list[i].f_keys[j].filename==NULL)
- {
- continue;
- }
- ret=stat(p->f_keys[j].filename, &file_info);
- if(ret==0)
- {
- continue;
- }
- snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key);
- ret=redisAppendCommand(ctx, redis_cmd);
- track[key_num].rule_idx=i;
- track[key_num].foreign_idx=j;
- key_num++;
- assert(ret==REDIS_OK);
- }
- }
- }
- for(i=0;itype!=REDIS_REPLY_STRING)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
- ,"Get %s,%d foreign key %s content failed."
- ,rule_list[track[i].rule_idx].table_name
- ,rule_list[track[i].rule_idx].rule_id
- ,rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
- continue;
- }
- else
- {
- p=rule_list+track[i].rule_idx;
- fp=fopen(p->f_keys[track[i].foreign_idx].filename, "w");
- if(fp==NULL)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
- , "Write foreign content failed: fopen %s error."
- , p->f_keys[track[i].foreign_idx]);
- }
- else
- {
- fwrite(reply->str, 1, reply->len, fp);
- fclose(fp);
- fp=NULL;
- if(print_fn==1)
- {
- printf("Written foreign content %s\n",p->f_keys[track[i].foreign_idx].filename);
- }
- }
- }
- freeReplyObject(reply);
- reply=NULL;
- }
-
- free(track);
- return;
-}
-void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger)
-{
- int max_redis_batch=4*1024,batch_cnt=0;
- int success_cnt=0;
- while(success_cntlogger;
-
- if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
- {
- //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
- if(1==redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout))
- {
- check_maat_expiration(mr_ctx->read_ctx, logger);
- cleanup_update_status(mr_ctx->read_ctx, logger);
- redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
- }
- }
- if(mr_ctx->read_ctx==NULL||mr_ctx->read_ctx->err)
- {
- if(time(NULL)-mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL)
- {
- return;
- }
- mr_ctx->last_reconnect_time=time(NULL);
- if(mr_ctx->read_ctx!=NULL)
- {
- redisFree(mr_ctx->read_ctx);
- }
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Reconnecting...");
- mr_ctx->read_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, feather->logger);
- if(mr_ctx->read_ctx==NULL)
- {
- return;
- }
- else
- {
- version=0;//Trigger full update when reconnect to redis.
- }
- }
-
- rule_num=get_rm_key_list(mr_ctx->read_ctx, version, feather->load_version_from, &new_version, feather->table_mgr, &rule_list, &update_type, logger, feather->cumulative_update_off);
- if(rule_num<0)//redis communication error
- {
- redisFree(mr_ctx->read_ctx);
- mr_ctx->read_ctx=NULL;
- return;
- }
- feather->load_version_from=0;//only valid for one time.
- if(rule_num==0&&update_type==CM_UPDATE_TYPE_INC)//error or nothing changed
- {
- return;
- }
- if(rule_num>0)
- {
- ret=get_maat_redis_value(mr_ctx->read_ctx, rule_list, rule_num, logger, 0);
- if(ret<0)//redis communication error
- {
- redisFree(mr_ctx->read_ctx);
- mr_ctx->read_ctx=NULL;
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Get Redis value failed, abandon update and close connection.");
- goto clean_up;
- }
- for(i=0;iread_ctx, rule_list, rule_num, feather, feather->foreign_cont_dir, logger);
- if(ret>0)
- {
- get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, logger);
- }
- }
- start(new_version, update_type, u_para);
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Start %s update: %lld -> %lld (%d entries).",
- update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num);
- for(i=0;itable_mgr, rule_list[i].table_name);
- if(table_id<0)//Unrecognized table.
- {
- no_table_num++;
- continue;
- }
- table_type=Maat_table_manager_get_type_by_id(feather->table_mgr, table_id);
- if(rule_list[i].op==MAAT_OP_DEL)
- {
-
- scan_type=Maat_table_get_scan_type(table_type);
- table_schema=Maat_table_manager_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL);
- valid_column=Maat_table_schema_get_valid_flag_column(table_schema);
- ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Invalidate line failed, invaid format %s .",
- rule_list[i].table_line);
- continue;
- }
- }
- if(rule_list[i].n_foreign>0)
- {
- rewrite_table_line_with_foreign(rule_list+i);
- }
- update(rule_list[i].table_name,rule_list[i].table_line,u_para);
- call_update_num++;
- }
- finish(u_para);
- if(call_update_numtable_name!=NULL)
- {
- dst->table_name=_maat_strdup(src->table_name);
- }
- switch(dst->region_type)
- {
- case REGION_IP:
- dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip);
- dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip);
- dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip);
- dst->ip_rule.mask_dst_ip=_maat_strdup(src->ip_rule.mask_dst_ip);
- break;
- case REGION_EXPR:
- dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords);
- dst->expr_rule.district=_maat_strdup(src->expr_rule.district);
- break;
- case REGION_INTERVAL:
- break;
- case REGION_DIGEST:
- dst->digest_rule.digest_string=_maat_strdup(src->digest_rule.digest_string);
- break;
- case REGION_SIMILARITY:
- dst->similarity_rule.target=_maat_strdup(src->similarity_rule.target);
- break;
- default:
- assert(0);
- }
- return;
-}
-void _maat_empty_region(struct Maat_region_t* p)
-{
- free((char*)p->table_name);
- p->table_name=NULL;
- switch(p->region_type)
- {
- case REGION_IP:
- free((char*)p->ip_rule.src_ip);
- free((char*)p->ip_rule.mask_src_ip);
- free((char*)p->ip_rule.dst_ip);
- free((char*)p->ip_rule.mask_dst_ip);
- break;
- case REGION_EXPR:
- free((char*)p->expr_rule.keywords);
- free((char*)p->expr_rule.district);
- break;
- case REGION_INTERVAL:
- break;
- case REGION_DIGEST:
- free((char*)p->digest_rule.digest_string);
- break;
- case REGION_SIMILARITY:
- free((char*)p->similarity_rule.target);
- break;
- default:
- assert(0);
- }
- memset(p,0,sizeof(struct Maat_region_t));
- return;
-
-}
-int Maat_command_raw_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, size_t n_line ,enum MAAT_OPERATION op)
-{
- size_t i=0;
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- int ret=0, table_id=0,success_cnt=0;
- struct serial_rule_t *s_rule=NULL;
- long long server_time=0,absolute_expire_time=0;
- redisContext* write_ctx=get_redis_ctx_for_write(_feather);
- if(write_ctx==NULL)
- {
- return -1;
- }
- server_time=redis_server_time(write_ctx);
- if(!server_time)
- {
- return -1;
- }
- s_rule=ALLOC(struct serial_rule_t, n_line);
- for(i=0;itable_mgr, line_rule[i]->table_name);
- if(table_id<0)
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,
- "Command raw set line id %d failed: unknown table %s.",
- line_rule[i]->rule_id,
- line_rule[i]->table_name);
- ret=-1;
- goto error_out;
- }
- if(op==MAAT_OP_RENEW_TIMEOUT)
- {
- assert(line_rule[i]->expire_after>0);
- }
- if(line_rule[i]->expire_after>0)
- {
- absolute_expire_time=server_time+line_rule[i]->expire_after;
- }
- set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
- line_rule[i]->table_line, absolute_expire_time);
- }
- success_cnt=exec_serial_rule(write_ctx,s_rule, n_line,server_time,_feather->logger);
- if(success_cnt<0||(size_t)success_cnt!=n_line)//error
- {
- ret=-1;
- goto error_out;
- }
- ret=success_cnt;
- _feather->line_cmd_acc_num+=success_cnt;
-
-error_out:
- for(i=0;itable_mgr, line_rule[i]->table_name);
- if(table_id<0)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
- ,"Command set line id %d failed: unknown table %s."
- , line_rule[i]->rule_id
- , line_rule[i]->table_name);
- ret=-1;
- goto error_out;
- }
- p_table=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id);
- if(!p_table)
- {
- ret=-1;
- goto error_out;
- }
- int valid_flag_column=0;
-
- valid_flag_column=Maat_table_schema_get_valid_flag_column(p_table);
- if(valid_flag_column<0)
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,
- "Command set line id %d failed: table %s is not a plugin or ip_plugin table.",
- line_rule[i]->rule_id,
- line_rule[i]->table_name);
- ret=-1;
- goto error_out;
-
- }
-
- if(op==MAAT_OP_ADD)
- {
- ret=get_valid_flag_offset(line_rule[i]->table_line,
- p_table->table_type,
- valid_flag_column);
- if(ret<0||
- (op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1'))
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,
- "Command set line %s %d failed: illegal valid flag.",
- line_rule[i]->table_name, line_rule[i]->rule_id);
- ret=-1;
- goto error_out;
- }
- }
- if(op==MAAT_OP_RENEW_TIMEOUT)
- {
- assert(line_rule[i]->expire_after>0);
- }
- if(line_rule[i]->expire_after>0)
- {
- absolute_expire_time=server_time+line_rule[i]->expire_after;
- }
- if(plugin_desc && plugin_desc->n_foreign>0)
- {
- for(j=0;jn_foreign;j++)
- {
- p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size);
- if(p_foreign==NULL)
- {
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command,
- "Command set line %s %d failed: No %dth column.",
- line_rule[i]->table_name, line_rule[i]->rule_id,
- plugin_desc->foreign_columns[j]);
- ret=-1;
- goto error_out;
- }
- if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_redis_monitor,
- "Command set line %s %d failed: Source prefix %s is mandatory.",
- line_rule[i]->table_name, line_rule[i]->rule_id, foreign_source_prefix);
- ret=-1;
- goto error_out;
- }
- }
-
- }
- set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
- line_rule[i]->table_line, absolute_expire_time);
- }
- success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger);
- if(success_cnt<0||success_cnt!=line_num)//error
- {
- ret=-1;
- goto error_out;
- }
- ret=success_cnt;
- _feather->line_cmd_acc_num+=success_cnt;
-
-error_out:
- for(i=0;imr_ctx.write_ctx;
- if(ctx==NULL)
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__);
- return -1;
- }
- const char *arg_vec[3];
- size_t len_vec[3];
- arg_vec[0] = "SET";
- len_vec[0] = strlen("SET");
-
- arg_vec[1] = key;
- len_vec[1] = strlen(key);
-
- arg_vec[2] = value;
- len_vec[2] = size;
-
- redisReply *reply=NULL;
- if(0!=strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix)))
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "Invalid File key, prefix %s is mandatory.", foreign_key_prefix);
- return -1;
- }
- switch(op)
- {
- case MAAT_OP_ADD:
- reply= (redisReply *)redisCommandArgv(ctx, sizeof(arg_vec) / sizeof(arg_vec[0]), arg_vec, len_vec);
- break;
- case MAAT_OP_DEL:
- reply=_wrap_redisCommand(ctx, "EXPIRE %s %d", key, MAAT_REDIS_SYNC_TIME);
- break;
- default:
- return -1;
- break;
- }
- if(reply==NULL||reply->type==REDIS_REPLY_NIL||reply->type==REDIS_REPLY_ERROR)
- {
- MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,"Set file failed, maybe Redis is busy.");
- freeReplyObject(reply);
- reply=NULL;
- return -1;
- }
- freeReplyObject(reply);
- reply=NULL;
- return 1;
-}
-
-long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
-{
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- redisReply* data_reply=NULL;
- long long result=0;
- redisContext* write_ctx=get_redis_ctx_for_write(_feather);
-
- if(write_ctx==NULL)
- {
- return -1;
- }
- data_reply=_wrap_redisCommand(write_ctx, "INCRBY %s %d", key, increment);
- if(data_reply->type==REDIS_REPLY_INTEGER)
- {
- result=data_reply->integer;
- }
- else
- {
- result=-1;
- }
- freeReplyObject(data_reply);
- data_reply=NULL;
- return result;
-}
-int Maat_command_get_new_group_id(Maat_feather_t feather)
-{
- int group_id=0;
- group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1);
- return group_id;
-}
-int Maat_command_get_new_region_id(Maat_feather_t feather)
-{
- int region_id=0;
- region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1);
- return region_id;
-}
-
-void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size)
-{
- int i=0;
- struct Maat_cmd_key* p=*keys;
- for(i=0; itable_name);
- p->table_name=NULL;
- p->rule_id=0;
- }
- free(*keys);
- *keys=NULL;
- return;
-}
-
-int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys)
-{
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- redisReply* data_reply=NULL;
- char* tmp=NULL;
- unsigned int i=0;
- struct Maat_cmd_key* result=NULL;
- int result_cnt=0;
- redisContext* write_ctx=get_redis_ctx_for_write(_feather);
- if(write_ctx==NULL)
- {
- return -1;
- }
-
- data_reply=_wrap_redisCommand(write_ctx,"ZRANGEBYSCORE %s %d %d",
- mr_label_sset,
- label_id,
- label_id);
- result_cnt=data_reply->elements;
- result=ALLOC(struct Maat_cmd_key, data_reply->elements);
- for(i=0;ielements;i++)
- {
- result[i].table_name=_maat_strdup(data_reply->element[i]->str);
- tmp=strchr(result[i].table_name, ',');
- if(tmp!=NULL)
- {
- *tmp='\0';
- tmp++;
- result[i].rule_id=atoi(tmp);
- }
- else// old version compatible
- {
- result[i].rule_id=atoi(result[i].table_name);
- free(result[i].table_name);
- result[i].table_name=NULL;
- }
- }
- freeReplyObject(data_reply);
- data_reply=NULL;
-
- *keys=result;
- return result_cnt;
-}
-
-int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
-{
- redisReply* data_reply=NULL;
- long long maat_redis_version=0, dbsize=0;
- int append_cmd_cnt=0, i=0,ret=0;
- int redis_transaction_success=1;
-
- data_reply=_wrap_redisCommand(ctx, "WATCH MAAT_VERSION");
- freeReplyObject(data_reply);
- data_reply=NULL;
- data_reply=_wrap_redisCommand(ctx, "GET MAAT_VERSION");
- if(data_reply->type==REDIS_REPLY_NIL)
- {
- maat_redis_version=0;
- }
- else
- {
- maat_redis_version=read_redis_integer(data_reply);
- maat_redis_version++;
- freeReplyObject(data_reply);
- data_reply=NULL;
- }
- data_reply=_wrap_redisCommand(ctx, "DBSIZE");
- dbsize=read_redis_integer(data_reply);
- freeReplyObject(data_reply);
- data_reply=NULL;
-
- data_reply=_wrap_redisCommand(ctx,"MULTI");
- freeReplyObject(data_reply);
- data_reply=NULL;
-
- redisAppendCommand(ctx,"FLUSHDB");
- append_cmd_cnt++;
- redisAppendCommand(ctx,"SET MAAT_VERSION %lld",maat_redis_version);
- append_cmd_cnt++;
- redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version);
- append_cmd_cnt++;
- redisAppendCommand(ctx,"SET %s 1", mr_region_id_var);
- append_cmd_cnt++;
- redisAppendCommand(ctx,"SET %s 1", mr_group_id_var);
- append_cmd_cnt++;
- redisAppendCommand(ctx,"EXEC");
- append_cmd_cnt++;
- for(i=0;iqueue);
- batch->feather=(struct _Maat_feather_t *)feather;
- redisContext* write_ctx=get_redis_ctx_for_write(batch->feather);
- if(write_ctx==NULL)
- {
- free(batch);
- return NULL;
- }
- batch->server_time=redis_server_time(write_ctx);
- if(!batch->server_time)
- {
- free(batch);
- return NULL;
- }
- return batch;
-}
-
-int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id)
-{
- struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
- long long absolute_expire_time=0;
- char line[MAX_TABLE_LINE_SIZE];
-
- serialize_region(region, group_id, line, sizeof(line));
-
- set_serial_rule(s_rule, op, region->region_id, 0, region->table_name,
- line, absolute_expire_time);
- TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
- batch->batch_size++;
- return 0;
-
-}
-#define TO_GROUP2X_KEY(group_id, parent_id) ((unsigned long)group_id<<32|parent_id)
-
-int Maat_command_batch_set_group2group(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g)
-{
- struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
- long long absolute_expire_time=0;
- char line[MAX_TABLE_LINE_SIZE];
-
- serialize_group2group(op, g2g, line, sizeof(line));
-
- set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2g->group_id, g2g->superior_group_id), 0, g2g->table_name,
- line, absolute_expire_time);
-
- TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
- batch->batch_size++;
- return 0;
-}
-int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c)
-{
- struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
- long long absolute_expire_time=0;
- char line[MAX_TABLE_LINE_SIZE];
-
- serialize_group2compile(op, g2c, line, sizeof(line));
- set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2c->group_id, g2c->compile_id), 0, g2c->table_name,
- line, absolute_expire_time);
- TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
- batch->batch_size++;
- return 0;
-}
-int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after)
-{
-
- struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
- long long absolute_expire_time=0;
- char line[MAX_TABLE_LINE_SIZE];
- serialize_compile(compile, huge_service_defined, clause_num, op, line, sizeof(line));
-
- if(expire_after>0)
- {
- absolute_expire_time=batch->server_time+expire_after;
- }
- set_serial_rule(s_rule, op, compile->config_id, label_id, table_name,
- line, absolute_expire_time);
-
- TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
- batch->batch_size++;
- return 0;
-}
-int Maat_command_batch_commit(struct Maat_command_batch* batch)
-{
- struct serial_rule_t* s_rule_array=ALLOC(struct serial_rule_t, batch->batch_size);
- int i=0;
- redisContext* write_ctx=get_redis_ctx_for_write(batch->feather);
- struct serial_rule_t * tmp = TAILQ_FIRST(&batch->queue);
-
- while(tmp != NULL)
- {
- TAILQ_REMOVE(&batch->queue, tmp, entries);
- memcpy(s_rule_array+i, tmp, sizeof(*tmp));
- free(tmp);
- tmp = TAILQ_FIRST(&batch->queue);
- i++;
- }
- assert(i==batch->batch_size);
- exec_serial_rule(write_ctx, s_rule_array, batch->batch_size, batch->server_time, batch->feather->logger);
- for(i=0; ibatch_size; i++)
- {
- empty_serial_rules(s_rule_array+i);
- }
- free(s_rule_array);
- free(batch);
- return i;
-}
-
-int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after)
-{
- struct Maat_command_batch* batch=NULL;
- batch=Maat_command_batch_new(feather);
- Maat_command_batch_set_compile(batch, op, compile, table_name, huge_service_defined, clause_num, label_id, expire_after);
- Maat_command_batch_commit(batch);
- return 0;
-}
-int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id)
-{
- struct Maat_command_batch* batch=NULL;
- batch=Maat_command_batch_new(feather);
- Maat_command_batch_set_region(batch, op, region, group_id);
- Maat_command_batch_commit(batch);
- return 0;
-}
-int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c)
-{
- struct Maat_command_batch* batch=NULL;
- batch=Maat_command_batch_new(feather);
- Maat_command_batch_set_group2compile(batch, op, g2c);
- Maat_command_batch_commit(batch);
- return 0;
-}
-int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g)
-{
-
- struct Maat_command_batch* batch=NULL;
- batch=Maat_command_batch_new(feather);
- Maat_command_batch_set_group2group(batch, op, g2g);
- Maat_command_batch_commit(batch);
- return 0;
-}
-
-int Maat_cmd_flushDB(Maat_feather_t feather)
-{
- _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
- int ret=0;
- redisContext* write_ctx=get_redis_ctx_for_write(_feather);
- if(write_ctx==NULL)
- {
- return -1;
- }
- do
- {
- ret=redis_flush_DB(_feather->mr_ctx.write_ctx, _feather->mr_ctx.redis_db, _feather->logger);
- }while(ret==0);
- return 0;
-}
-
diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp
deleted file mode 100644
index ec73c61..0000000
--- a/src/entry/Maat_ex_data.cpp
+++ /dev/null
@@ -1,397 +0,0 @@
-#include "Maat_ex_data.h"
-#include "Maat_table_schema.h"
-#include "Maat_utils.h"
-#include "Maat_garbage_collection.h"
-
-#include "uthash/uthash.h"
-#include "uthash/utarray.h"
-
-#include
-#include
-
-
-struct EX_data_container
-{
- void* ex_data;
- char* key;
- size_t key_len;
- void* user_data;
-
- UT_hash_handle hh_a, hh_b;
- const struct EX_data_rt* rt;
-
-};
-
-struct EX_data_rt
-{
- UT_array *cache_rows;
- size_t cache_row_num;
- size_t cache_size;
-
- pthread_mutex_t mutex_update_commit;
- int is_updating;
- char effective_hash;//value 'a' or 'b', indicates which hash is effective
- struct EX_data_container* hash_key2ex_a, *hash_key2ex_b; //two hash for read-copy-update (RCU)
-
- const struct EX_data_schema* ex_schema;
- int table_id;
- void (* user_data_free)(void *user_data);
- struct Maat_garbage_bin* garbage_bin;
-};
-
-
-void EX_data_container_free(struct EX_data_container* ex_container)
-{
- const struct EX_data_schema* ex_schema=ex_container->rt->ex_schema;
- ex_schema->free_func(ex_container->rt->table_id, &(ex_container->ex_data), ex_schema->argl, ex_schema->argp);
- if(ex_container->user_data && ex_container->rt->user_data_free)
- {
- ex_container->rt->user_data_free(ex_container->user_data);
- }
- free(ex_container->key);
- ex_container->key=NULL;
- ex_container->key_len=0;
- ex_container->user_data=NULL;
- ex_container->rt=NULL;
- free(ex_container);
- return;
-}
-void cache_row_free(void*p)
-{
- free(*(char**)p);
-}
-UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
-
-struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
-{
- struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
- p->hash_key2ex_a=NULL;
- p->hash_key2ex_b=NULL;
- p->effective_hash='a';
- utarray_new(p->cache_rows, &ut_cache_row_icd);
- p->table_id=table_id;
- p->user_data_free=user_data_free;
- p->garbage_bin=Maat_garbage_bin_new(60);
- pthread_mutex_init(&p->mutex_update_commit, NULL);
- return p;
-};
-size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt)
-{
- return ex_rt->cache_row_num;
-}
-void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema)
-{
- p->ex_schema=schema;
-}
-void EX_data_rt_free(struct EX_data_rt* ex_rt)
-{
- struct EX_data_container* ex_container=NULL, *tmp=NULL;
-
- assert(ex_rt->is_updating==0);
- if(ex_rt->effective_hash=='a')
- {
- HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp)
- {
- HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, ex_container);
- EX_data_container_free(ex_container);
- }
- }
- else
- {
- HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp)
- {
- HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, ex_container);
- EX_data_container_free(ex_container);
- }
-
- }
- //ex_rt->hash_key2ex's memory is freed when its last element was deleted.
-
- if(ex_rt->cache_rows)
- {
- utarray_free(ex_rt->cache_rows);
- ex_rt->cache_rows=NULL;
- ex_rt->cache_row_num=0;
- }
- Maat_garbage_bin_free(ex_rt->garbage_bin);
- ex_rt->garbage_bin=NULL;
- free(ex_rt);
- return;
-}
-void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row)
-{
- size_t len=strlen(row)+1;
- char* row_copy=ALLOC(char, len);
- memcpy(row_copy, row, len);
- p->cache_size+=len;
- utarray_push_back(p->cache_rows, &row_copy);
- p->cache_row_num++;
- return;
-}
-const char* EX_data_rt_cached_row_get(struct EX_data_rt* p, size_t index)
-{
- const char** row=NULL;
- row=(const char**)utarray_eltptr(p->cache_rows, index);
- return *row;
-}
-void EX_data_rt_clear_row_cache(struct EX_data_rt* p)
-{
- utarray_free(p->cache_rows);
- p->cache_rows=NULL;
- p->cache_row_num=0;
- p->cache_size=0;
-}
-void EX_data_rt_update_prepare(struct EX_data_rt* ex_rt)
-{
- struct EX_data_container* ex_container=NULL, *tmp=NULL;
- if(ex_rt->effective_hash=='a')
- {
- assert(ex_rt->hash_key2ex_b==NULL);
- HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp)
- {
- HASH_ADD_KEYPTR(hh_b, ex_rt->hash_key2ex_b, ex_container->key, ex_container->key_len, ex_container);
- }
- }
- else
- {
- assert(ex_rt->hash_key2ex_a==NULL);
- HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp)
- {
- HASH_ADD_KEYPTR(hh_a, ex_rt->hash_key2ex_a, ex_container->key, ex_container->key_len, ex_container);
- }
- }
- assert(0==Maat_garbage_bin_get_size(ex_rt->garbage_bin));
- ex_rt->is_updating=1;
- return;
-}
-
-void EX_data_rt_update_commit(struct EX_data_rt* ex_rt)
-{
- struct EX_data_container* ex_container=NULL, *tmp=NULL;
- //lock contention of Maat_plugin_EX_register and rule update thread.
- pthread_mutex_lock(&ex_rt->mutex_update_commit);
- if(!ex_rt->is_updating)
- {
- pthread_mutex_unlock(&ex_rt->mutex_update_commit);
- return;
- }
- if(ex_rt->effective_hash=='a')
- {
- ex_rt->effective_hash='b';
- usleep(100); //urgly sleep, wait for EX_data_rt_get_EX_data_by_key to release the effective hash table.
- HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp)
- {
- HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, ex_container);
- //ex_container is no need to free, it's either in new hash table or garbage collection queue.
- }
- }
- else
- {
- ex_rt->effective_hash='a';
- usleep(100);
- HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp)
- {
- HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, ex_container);
- }
- }
- ex_rt->is_updating=0;
- Maat_garbage_collect_by_force(ex_rt->garbage_bin);
- pthread_mutex_unlock(&ex_rt->mutex_update_commit);
- return;
-}
-
-struct EX_data_container* EX_data_rt_effective_hash_find(struct EX_data_rt* ex_rt, const char* key, size_t key_len)
-{
- struct EX_data_container* exc=NULL;
-
- if(ex_rt->effective_hash=='a')
- {
- HASH_FIND(hh_a, ex_rt->hash_key2ex_a, key, key_len, exc);
- }
- else
- {
- assert(ex_rt->effective_hash=='b');
- HASH_FIND(hh_b, ex_rt->hash_key2ex_b, key, key_len, exc);
- }
- return exc;
-}
-
-struct EX_data_container* EX_data_rt_updating_hash_find(struct EX_data_rt* ex_rt, const char* key, size_t key_len)
-{
- struct EX_data_container* exc=NULL;
-
- if(ex_rt->effective_hash=='a')
- {
- HASH_FIND(hh_b, ex_rt->hash_key2ex_b, key, key_len, exc);
- }
- else
- {
- assert(ex_rt->effective_hash=='b');
- HASH_FIND(hh_a, ex_rt->hash_key2ex_a, key, key_len, exc);
- }
- return exc;
-}
-void EX_data_rt_updating_hash_add(struct EX_data_rt* ex_rt, const char* key, size_t key_len, struct EX_data_container* exc)
-{
- if(ex_rt->effective_hash=='a')
- {
- HASH_ADD_KEYPTR(hh_b, ex_rt->hash_key2ex_b, key,key_len, exc);
- }
- else
- {
- HASH_ADD_KEYPTR(hh_a, ex_rt->hash_key2ex_a, key,key_len, exc);
- }
- return;
-}
-void EX_data_rt_updating_hash_del(struct EX_data_rt* ex_rt, struct EX_data_container* exc)
-{
- if(ex_rt->effective_hash=='a')
- {
- HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, exc);
- }
- else
- {
- HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, exc);
- }
- return;
-}
-int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt,
- const char* row, const char* key, size_t key_len,
- void* user_data, void* logger)
-{
-
- MAAT_RULE_EX_DATA ex_data=NULL;
- const struct EX_data_schema* ex_schema=ex_rt->ex_schema;
- struct EX_data_container* ex_container=NULL, *tmp=NULL;
- int ret=0;
- if(!ex_rt->is_updating)
- {
- EX_data_rt_update_prepare(ex_rt);
- }
- tmp=EX_data_rt_updating_hash_find(ex_rt, key, key_len);
- if(tmp==NULL)
- {
- ex_container=ALLOC(struct EX_data_container, 1);
- ex_container->key=ALLOC(char, key_len+1);
- memcpy(ex_container->key, key, key_len);
- ex_schema->new_func(ex_rt->table_id, ex_container->key, row, &ex_data,
- ex_schema->argl, ex_schema->argp);
- ex_container->ex_data=ex_data;
- ex_container->rt=ex_rt;
- ex_container->user_data=user_data;
-
- ex_container->key_len=key_len;
- EX_data_rt_updating_hash_add(ex_rt, ex_container->key, ex_container->key_len, ex_container);
- ret=0;
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "EX data add error: duplicated key %.*s of %s",
- key_len, key, row);
-
- ret=-1;
- }
- return ret;
-}
-int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len,
- void *logger)
-{
- struct EX_data_container* exc=NULL;
- int ret=0;
- if(!ex_rt->is_updating)
- {
- EX_data_rt_update_prepare(ex_rt);
- }
- exc=EX_data_rt_updating_hash_find(ex_rt, key, key_len);
- if(exc)
- {
- EX_data_rt_updating_hash_del(ex_rt, exc);
- Maat_garbage_bagging(ex_rt->garbage_bin, exc, (void (*)(void*))EX_data_container_free);
- ret=0;
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "EX data del error: no such key %.*s of %s",
- key_len, key, row);
- ret=-1;
- }
- return ret;
-}
-
-MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len)
-{
- struct EX_data_container* exc=NULL;
- MAAT_RULE_EX_DATA ex_data=NULL;
-
- if(!ex_rt->ex_schema)
- {
- assert(0);
- return NULL;
- }
- exc=EX_data_rt_effective_hash_find(ex_rt, key, key_len);
- if(exc!=NULL)
- {
- ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(exc->ex_data),
- ex_rt->ex_schema->argl, ex_rt->ex_schema->argp);
- }
- return ex_data;
-}
-MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container)
-{
- MAAT_RULE_EX_DATA dupped_ex_data=NULL;
- ex_rt->ex_schema->dup_func(ex_rt->table_id, &(dupped_ex_data), &(container->ex_data),
- ex_rt->ex_schema->argl, ex_rt->ex_schema->argp);
- return dupped_ex_data;
-}
-
-size_t EX_data_rt_list_updating_ex_containers(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array)
-{
- size_t ex_data_cnt=0, i=0;
- struct EX_data_container* ex_container=NULL, *tmp=NULL;
- if(!ex_rt->ex_schema)
- {
- return 0;
- }
- assert(ex_rt->is_updating==1);
- if(ex_rt->effective_hash=='a')
- {
- ex_data_cnt=HASH_CNT(hh_b, ex_rt->hash_key2ex_b);
- *ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt);
- HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp)
- {
- (*ex_container_array)[i]=ex_container;
- i++;
- }
- }
- else
- {
- ex_data_cnt=HASH_CNT(hh_a, ex_rt->hash_key2ex_a);
- *ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt);
- HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp)
- {
- (*ex_container_array)[i]=ex_container;
- i++;
- }
- }
- return ex_data_cnt;
-}
-void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
-{
- return ex_container->user_data;
-}
-size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
-{
- size_t count=0;
- if(ex_rt->effective_hash=='a')
- {
- count=HASH_CNT(hh_a, ex_rt->hash_key2ex_a);
- }
- else
- {
- count=HASH_CNT(hh_b, ex_rt->hash_key2ex_b);
- }
- return count;
-}
-
diff --git a/src/entry/Maat_garbage_collection.cpp b/src/entry/Maat_garbage_collection.cpp
deleted file mode 100644
index ff981bf..0000000
--- a/src/entry/Maat_garbage_collection.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-
-#include "Maat_utils.h"
-
-#include
-#include
-#include
-#include
-
-struct Maat_garbage_bag
-{
- time_t create_time;
- int timeout;
- int ok_times;
- void *garbage;
- void (* garbage_free)(void *garbage);
- TAILQ_ENTRY(Maat_garbage_bag) entries;
-};
-TAILQ_HEAD(Maat_garbage_q, Maat_garbage_bag);
-
-struct Maat_garbage_bin
-{
- Maat_garbage_q garbage_q;
- size_t bag_cnt;
- int timeout_seconds;
-};
-struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout)
-{
- struct Maat_garbage_bin* bin=ALLOC(struct Maat_garbage_bin, 1);
- TAILQ_INIT(&bin->garbage_q);
- bin->timeout_seconds=default_timeout;
- return bin;
-}
-void Maat_garbage_bin_free(struct Maat_garbage_bin* bin)
-{
- struct Maat_garbage_bag* p=NULL;
- while (p=TAILQ_FIRST(&bin->garbage_q))
- {
- p->garbage_free(p->garbage);
- TAILQ_REMOVE(&bin->garbage_q, p, entries);
- free(p);
- bin->bag_cnt--;
- }
-
- free(bin);
- return;
-}
-size_t Maat_garbage_bin_get_size(struct Maat_garbage_bin* bin)
-{
- return bin->bag_cnt;
-}
-void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *))
-{
- struct Maat_garbage_bag* bag=ALLOC( struct Maat_garbage_bag, 1);
- bag->create_time=time(NULL);
- bag->timeout=bin->timeout_seconds;
- bag->garbage=garbage;
- bag->garbage_free=func;
- TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
- bin->bag_cnt++;
-}
-void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin)
-{
- struct Maat_garbage_bag* p=NULL, *tmp=NULL;
- size_t n_clollected=0, n_bag=0;
- time_t now=time(NULL);
-
- for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp)
- {
- tmp=TAILQ_NEXT(p, entries);
- if(now-p->create_time>p->timeout || p->timeout==0)
- {
- p->garbage_free(p->garbage);
- TAILQ_REMOVE(&bin->garbage_q, p, entries);
- free(p);
- n_clollected++;
- }
- n_bag++;
- }
-
- assert(bin->bag_cnt==n_bag);
- bin->bag_cnt-=n_clollected;
- return;
-}
-void Maat_garbage_collect_by_force(struct Maat_garbage_bin* bin)
-{
- struct Maat_garbage_bag* p=NULL;
- while (p=TAILQ_FIRST(&bin->garbage_q))
- {
- p->garbage_free(p->garbage);
- TAILQ_REMOVE(&bin->garbage_q, p, entries);
- free(p);
- bin->bag_cnt--;
- }
- return;
-}
-
diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp
deleted file mode 100644
index 2188c6e..0000000
--- a/src/entry/Maat_hierarchy.cpp
+++ /dev/null
@@ -1,1550 +0,0 @@
-
-#include "Maat_hierarchy.h"
-#include "Maat_utils.h"
-#include "Maat_limits.h"
-#include "uthash/uthash.h"
-#include "uthash/utarray.h"
-#include "igraph/igraph.h"
-#include "bool_matcher.h"
-
-#include
-
-#include
-#include
-
-#define module_maat_hierarchy "MAAT_HIERARCHY"
-
-
-
-struct Maat_hierarchy_group
-{
- igraph_integer_t vertex_id;
- int group_id;
- int ref_by_compile_cnt;
- int ref_by_superior_group_cnt;
- int ref_by_subordinate_group_cnt;
- int ref_by_region_cnt;
-
- size_t top_group_cnt;
- int* top_group_ids;
- UT_hash_handle hh_group_id;
- UT_hash_handle hh_vertex_id;
-};
-struct Maat_hierarchy_region
-{
- int region_id;
- int group_id;
- int table_id;
-
- struct Maat_hierarchy_group* ref_parent_group;
- UT_hash_handle hh;
-
- void* user_data;
-};
-
-struct Maat_hierarchy_literal_id
-{
- int group_id;
- int vt_id;
-};
-
-struct Maat_hierarchy_literal
-{
- struct Maat_hierarchy_literal_id literal_id;
- UT_array *clause_ids;
- UT_hash_handle hh; //index to
-};
-struct Maat_hierarchy_clause_state
-{
-
- unsigned long long clause_id;
- char not_flag;
- char in_use;
- UT_array *literal_ids;
-};
-
-struct Maat_hierarchy_internal_hit_path
-{
- int Nth_scan;
- int Nth_hit_region;
- int region_id;
- int virtual_table_id;
-};
-static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
-{
- struct Maat_hierarchy_internal_hit_path new_path;
- new_path.region_id=region_id;
- new_path.Nth_hit_region=Nth_region_result;
- new_path.Nth_scan=Nth_scan;
- new_path.virtual_table_id=virtual_table_id;
- /*
- struct Maat_hierarchy_internal_hit_path *tmp_path=NULL;
- size_t i=0, num=utarray_len(hit_paths);
- for(i=0; iNth_scan!=new_path.Nth_scan)
- {
- break;
- }
- else
- {
- if(tmp_path->region_id==new_path.region_id &&
- tmp_path->virtual_table_id==new_path.virtual_table_id &&
- tmp_path->Nth_hit_region==new_path.Nth_hit_region)
- {
- return 0;
- }
- }
- }
- */
- utarray_push_back(hit_paths, &new_path);
- return 1;
-}
-
-UT_icd ut_literal_id_icd = {sizeof(struct Maat_hierarchy_literal_id), NULL, NULL, NULL};
-UT_icd ut_clause_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
-UT_icd ut_region_id_icd = {sizeof(int), NULL, NULL, NULL};
-UT_icd ut_hit_path_icd = {sizeof(struct Maat_hierarchy_internal_hit_path), NULL, NULL, NULL};
-
-#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d
-struct Maat_hierarchy_compile
-{
- unsigned int magic;
- int compile_id;
- int actual_clause_num;
- int declared_clause_num;
- int not_clause_cnt;
- void* user_data;
- UT_hash_handle hh;
- struct Maat_hierarchy_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR];
-};
-
-static void _group_vertex_free(struct Maat_hierarchy_group* group)
-{
- free(group->top_group_ids);
- free(group);
-}
-
-struct Maat_hierarchy_clause
-{
- long long clause_id;
- size_t n_literal_id;
- struct Maat_hierarchy_literal_id* literal_ids;
- UT_hash_handle hh;
-};
-
-struct group2region
-{
- int group_id;
- UT_array* region_ids;
- UT_hash_handle hh; //index to
-};
-struct region2clause_key
-{
- int region_id;
- int vt_id;
-};
-struct region2clause_value
-{
- struct region2clause_key key;
- UT_array* clause_ids;
- int group_id;
- UT_hash_handle hh; //index to
-};
-void Maat_hierarchy_free_region2clause_hash(struct region2clause_value* hash)
-{
- struct region2clause_value* r2c_val=NULL, *tmp_r2c_val=NULL;
- HASH_ITER(hh, hash, r2c_val, tmp_r2c_val)
- {
- HASH_DEL(hash, r2c_val);
- utarray_free(r2c_val->clause_ids);
- free(r2c_val);
- }
- assert(hash==NULL);
- return;
-}
-
-struct Maat_hierarchy
-{
- pthread_rwlock_t rwlock;
- pthread_mutex_t mutex;
- time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match.
- int changed_flag;
-
- struct Maat_hierarchy_compile* hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*.
- void (* compile_user_data_free)(void *compile_ud);
-
- struct Maat_hierarchy_group* hash_group_by_id; //key: group_id, value: struct Maat_hierarchy_group*.
- struct Maat_hierarchy_group* hash_group_by_vertex; //key:vetex_id, value: struct Maat_hierarchy_group*. Multimap (Items with multiple keys).
-
-
- struct Maat_hierarchy_region* hash_region_by_id; //key: region_id, value: struct Maat_hierarchy_region*.
-
- struct Maat_hierarchy_clause* hash_dedup_clause_by_literals; //key: literal combination, value: struct Maat_hierarchy_clause*. For generating unique clause_id.
- unsigned long long clause_id_generator; //Increasing number.
-
- void (* region_user_data_free)(void *region_ud);
-
-
- igraph_t group_graph;
- igraph_integer_t group_graph_vcount;
- igraph_vector_t dfs_vids;
-
- igraph_integer_t grp_vertex_id_generator;
-
- /*Following members are accessed from scan threads.*/
- struct region2clause_value* hash_region2clause; //key: region_id+virtual_table_id, value: struct region2clause_value.
- struct bool_matcher* bm;
-
-
- int thread_num;
- struct Maat_garbage_bin* ref_garbage_bin;
- void* logger;
- struct bool_expr_match *expr_match_buff;
-};
-
-int compare_literal_id(const void *pa, const void *pb)
-{
- struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa;
- struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb;
- int ret=la->vt_id-lb->vt_id;
- if(ret==0)
- {
- ret=la->group_id-lb->group_id;
- }
- return ret;
-
-}
-static inline int compare_clause_id(const void* a, const void* b)
-{
- long long ret=*(const unsigned long long *)a - *(const unsigned long long *)b;
- if(ret==0)
- {
- return 0;
- }
- else if(ret<0)
- {
- return -1;
- }
- else
- {
- return 1;
- }
-}
-static inline int compare_region_id(const void* a, const void* b)
-{
- int ret= *(int*)a - *(int*)b;
- return ret;
-}
-
-static const struct Maat_hierarchy_clause* Maat_hierarchy_clause_fetch(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal_id* literal_ids, size_t n_literal_id)
-{
- struct Maat_hierarchy_clause* clause=NULL;
-
- HASH_FIND(hh, hier->hash_dedup_clause_by_literals, literal_ids,
- n_literal_id*sizeof(struct Maat_hierarchy_literal_id), clause);
-
- if(!clause)
- {
- clause=ALLOC(struct Maat_hierarchy_clause, 1);
- clause->clause_id=hier->clause_id_generator;
- clause->n_literal_id=n_literal_id;
- clause->literal_ids=ALLOC(struct Maat_hierarchy_literal_id, n_literal_id);
- memcpy(clause->literal_ids, literal_ids, n_literal_id*sizeof(struct Maat_hierarchy_literal_id));
-
- hier->clause_id_generator++;
- HASH_ADD_KEYPTR(hh, hier->hash_dedup_clause_by_literals, clause->literal_ids,
- n_literal_id*sizeof(struct Maat_hierarchy_literal_id),
- clause);
- }
- return clause;
-}
-static void Maat_hierarchy_clause_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_clause* clause)
-{
- HASH_DELETE(hh, hier->hash_dedup_clause_by_literals, clause);
- free(clause->literal_ids);
- clause->n_literal_id=0;
- free(clause);
- return;
-}
-
-static int Maat_hierarchy_compile_add_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id, int not_flag, int clause_index)
-{
- struct Maat_hierarchy_clause_state* clause_state=compile->clause_states+clause_index;
- struct Maat_hierarchy_literal_id* tmp=NULL;
- clause_state->not_flag=not_flag;
- if(!clause_state->in_use)
- {
- clause_state->in_use=1;
- compile->actual_clause_num++;
- }
- tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id);
- if(tmp)
- {
- assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id));
- return -1;
- }
- else
- {
- utarray_push_back(clause_state->literal_ids, literal_id);
- utarray_sort(clause_state->literal_ids, compare_literal_id);
- }
- return 0;
-}
-static int Maat_hierarchy_compile_remove_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id, int clause_index)
-{
- struct Maat_hierarchy_clause_state* clause_state=compile->clause_states+clause_index;
- struct Maat_hierarchy_literal_id* tmp=NULL;
- size_t remove_idx=0;
- tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id , compare_literal_id);
- if(tmp)
- {
- assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id));
- }
- else
- {
- return -1;
- }
- remove_idx=utarray_eltidx(clause_state->literal_ids, tmp);
- utarray_erase(clause_state->literal_ids, remove_idx, 1);
- if(0==utarray_len(clause_state->literal_ids))
- {
- clause_state->in_use=0;
- compile->actual_clause_num--;
- }
- return 0;
-}
-static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int compile_id)
-{
- int i=0;
- struct Maat_hierarchy_compile* compile=NULL;
- compile=ALLOC(struct Maat_hierarchy_compile, 1);
- compile->magic=MAAT_HIER_COMPILE_MAGIC;
- compile->compile_id=compile_id;
- HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile);
- for(i=0; iclause_states[i].literal_ids, &ut_literal_id_icd);
- compile->clause_states[i].in_use=0;
- }
- return compile;
-}
-static void Maat_hierarchy_compile_free(struct Maat_hierarchy_compile* compile)
-{
- int i=0;
- struct Maat_hierarchy_clause_state* clause_state=NULL;
- //user_data must be freed before calling this function.
- assert(compile->user_data==NULL);
- for(i=0; iclause_states+i;
- utarray_free(clause_state->literal_ids);
- clause_state->literal_ids=NULL;
- clause_state->in_use=0;
- }
- compile->magic=0;
- free(compile);
-}
-
-static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data)
-{
- struct Maat_hierarchy_region* region=NULL;
- region=ALLOC(struct Maat_hierarchy_region, 1);
- region->group_id=group_id;
- region->region_id=region_id;
- region->table_id=table_id;
- region->ref_parent_group=parent_group;
- region->user_data=user_data;
- HASH_ADD_INT(hier->hash_region_by_id, region_id, region);
- parent_group->ref_by_region_cnt++;
- return region;
-}
-static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region)
-{
- HASH_DELETE(hh, hier->hash_region_by_id, region);
- region->ref_parent_group->ref_by_region_cnt--;
- if(hier->region_user_data_free && region->user_data)
- {
- hier->region_user_data_free(region->user_data);
- region->user_data=NULL;
- }
- free(region);
- return;
-}
-
-
-struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger, struct Maat_garbage_bin* bin)
-{
- struct Maat_hierarchy* hier=ALLOC(struct Maat_hierarchy, 1);
- UNUSED int ret=0;
- hier->logger=mesa_handle_logger;
- hier->thread_num=thread_num;
- hier->version=time(NULL);
-
- hier->hash_group_by_id=NULL;
- hier->hash_group_by_vertex=NULL;
- hier->hash_compile_by_id=NULL;
- hier->hash_region2clause=NULL;
- hier->hash_region_by_id=NULL;
- hier->hash_dedup_clause_by_literals=NULL;
- hier->clause_id_generator=0;
- hier->ref_garbage_bin=bin;
- hier->expr_match_buff=ALLOC(struct bool_expr_match, thread_num*MAX_SCANNER_HIT_NUM);
-
- pthread_mutex_init(&hier->mutex, NULL);
- ret=pthread_rwlock_init(&hier->rwlock, NULL);
- assert(ret==0);
- ret=igraph_empty(&hier->group_graph, 0, IGRAPH_DIRECTED);
- assert(ret==IGRAPH_SUCCESS);
- return hier;
-}
-void Maat_hierarchy_free(struct Maat_hierarchy* hier)
-{
- struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
- struct Maat_hierarchy_group* group=NULL, *tmp_group=NULL;
- struct region2clause_value* r2c_val=NULL, *tmp_r2c_val=NULL;
- struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL;
- struct Maat_hierarchy_clause* clause=NULL, *tmp_clause=NULL;
- pthread_rwlock_wrlock(&hier->rwlock);
-
- //Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do
- //Some have asked how uthash cleans up its internal memory.
- //The answer is simple: when you delete the final item from a hash table,
- //uthash releases all the internal memory associated with that hash table,
- //and sets its pointer to NULL.
- HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
- {
- if(hier->compile_user_data_free && compile->user_data)
- {
- hier->compile_user_data_free(compile->user_data);
- compile->user_data=NULL;
- }
- HASH_DEL(hier->hash_compile_by_id, compile);
- Maat_hierarchy_compile_free(compile);
- }
- assert(hier->hash_compile_by_id==NULL);
-
- HASH_ITER(hh, hier->hash_region2clause, r2c_val, tmp_r2c_val)
- {
- HASH_DEL(hier->hash_region2clause, r2c_val);
- utarray_free(r2c_val->clause_ids);
- free(r2c_val);
- }
- Maat_hierarchy_free_region2clause_hash(hier->hash_region2clause);
-
- HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region)
- {
- Maat_hierarchy_region_free(hier, region);
- }
-
- HASH_ITER(hh, hier->hash_dedup_clause_by_literals, clause, tmp_clause)
- {
- Maat_hierarchy_clause_free(hier, clause);
- }
-
- //Free group as the last.
- HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up.
- HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group)
- {
- HASH_DELETE(hh_group_id, hier->hash_group_by_id, group);
- _group_vertex_free(group);
- }
- assert(hier->hash_group_by_id==NULL);
-
- igraph_destroy(&hier->group_graph);
- bool_matcher_free(hier->bm);
- hier->bm=NULL;
- pthread_rwlock_unlock(&hier->rwlock);
- pthread_rwlock_destroy(&hier->rwlock);
- free(hier->expr_match_buff);
- hier->expr_match_buff=NULL;
- free(hier);
-}
-
-void Maat_hierarchy_set_compile_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *))
-{
- hier->compile_user_data_free=func;
- return;
-}
-void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *))
-{
- hier->region_user_data_free=func;
- return;
-}
-
-
-int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data)
-{
- int ret=0;
- struct Maat_hierarchy_compile* compile=NULL;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
- if(!compile)
- {
- assert(declared_clause_num>=0);
- compile=Maat_hierarchy_compile_new(hier, compile_id);
- compile->declared_clause_num=declared_clause_num;
- compile->user_data=user_data;
- }
- else
- {
- if(compile->user_data!=NULL)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Add compile %d failed, compile is already exisited.",
- compile_id);
- ret=-1;
- }
- else
- {
- compile->declared_clause_num=declared_clause_num;
- compile->user_data=user_data;
- }
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-}
-int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
-{
- struct Maat_hierarchy_compile* compile=NULL;
- int ret=0;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
- if(compile)
- {
- if(hier->compile_user_data_free && compile->user_data)
- {
- hier->compile_user_data_free(compile->user_data);
- compile->user_data=NULL;
- }
- if(compile->actual_clause_num==0)
- {
- HASH_DEL(hier->hash_compile_by_id, compile);
- Maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))Maat_hierarchy_compile_free);
- }
- ret=0;
- }
- else
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove compile %d failed, compile is not exisited.",
- compile_id);
- ret=-1;
-
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-}
-static void* Maat_hier_compile_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach)
-{
- struct Maat_hierarchy_compile* compile=NULL;
- void* ret=NULL;
-
- pthread_rwlock_rdlock(&hier->rwlock);
- HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
- if(compile)
- {
- ret=compile->user_data;
- if(is_dettach)
- {
- compile->user_data=NULL;
- }
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-
-}
-void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id)
-{
- void* user_data=NULL;
- user_data=Maat_hier_compile_get_user_data(hier, compile_id, 1);
- return user_data;
-}
-void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id)
-{
- void* user_data=NULL;
- user_data=Maat_hier_compile_get_user_data(hier, compile_id, 0);
- return user_data;
-
-}
-void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param)
-{
- struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
- pthread_rwlock_rdlock(&hier->rwlock);
- HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
- {
- if(compile->user_data)
- {
- callback(compile->user_data, param);
- }
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return;
-}
-struct Maat_hierarchy_group* Maat_hierarchy_group_new(struct Maat_hierarchy* hier, int group_id)
-{
- struct Maat_hierarchy_group* group=NULL;
- group=ALLOC(struct Maat_hierarchy_group, 1);
- group->group_id=group_id;
- group->vertex_id=hier->grp_vertex_id_generator++;
- assert(igraph_vcount(&hier->group_graph)==group->vertex_id);
- igraph_add_vertices(&hier->group_graph, 1, NULL); //Add 1 vertice.
-
- HASH_ADD(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group->group_id), group);
- HASH_ADD(hh_vertex_id, hier->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group);
-
- return group;
-}
-void vector_print(igraph_vector_t *v) {
- long int i;
- for (i=0; iref_by_compile_cnt==0&&group->ref_by_superior_group_cnt==0);
- igraph_vector_init(&v, 8);
- igraph_neighbors(&hier->group_graph, &v, group->vertex_id, IGRAPH_ALL);
- if(igraph_vector_size(&v)>0)
- {
- print_igraph_vector(&v, buff, sizeof(buff));
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Del group %d exception, still reached by %s.",
- group->vertex_id, buff);
- assert(0);
- }
- igraph_vector_destroy(&v);
- assert(group->top_group_ids==NULL);
- //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices.
- //igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id));
-
-
- HASH_DELETE(hh_group_id, hier->hash_group_by_id, group);
- HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group);
-
- _group_vertex_free(group);
-
- return;
-}
-
-int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id)
-{
- int ret=0;
- struct Maat_hierarchy_group* group=NULL;
- struct Maat_hierarchy_literal_id literal_id={group_id, vt_id};
- struct Maat_hierarchy_compile* compile=NULL;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(!group)
- {
- group=Maat_hierarchy_group_new(hier, group_id);
- }
-
- HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile);
- if(!compile)
- {
- compile=Maat_hierarchy_compile_new(hier, compile_id);
- }
-
- ret=Maat_hierarchy_compile_add_literal(compile, &literal_id, not_flag, clause_index);
- if(ret<0)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.",
- group_id, vt_id, clause_index, compile_id);
- ret=-1;
- }
- else
- {
- ret=0;
- group->ref_by_compile_cnt++;
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-}
-int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id)
-{
- struct Maat_hierarchy_group* group=NULL;
- struct Maat_hierarchy_literal_id literal_id={group_id, vt_id};
- struct Maat_hierarchy_compile* compile=NULL;
- int ret=0;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(!group)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove group %d from compile %d failed, group is not exisited.",
- group_id, compile_id);
- goto error_out;
- }
- HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile);
- if(!compile)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove group %d from compile %d failed, compile is not exisited.",
- group_id, compile_id);
- goto error_out;
- }
-
- ret=Maat_hierarchy_compile_remove_literal(compile, &literal_id, clause_index);
- if(ret<0)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.",
- group_id, vt_id, clause_index, compile_id);
- goto error_out;
- }
- if(compile->actual_clause_num==0 && !compile->user_data)
- {
- HASH_DEL(hier->hash_compile_by_id, compile);
- Maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))Maat_hierarchy_compile_free);
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return 0;
-
-error_out:
- pthread_rwlock_unlock(&hier->rwlock);
- return -1;
-}
-
-
-int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id)
-{
- int ret=0;
- igraph_integer_t edge_id;
- struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(!group)
- {
- group=Maat_hierarchy_group_new(hier, group_id);
- }
-
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group);
- if(!superior_group)
- {
- superior_group=Maat_hierarchy_group_new(hier, superior_group_id);
- }
- ret=igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
- if(edge_id>0)//No duplicated edges between two groups.
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Add group %d to group %d failed, relation already exisited.",
- group->group_id, superior_group->group_id);
- ret=-1;
- }
- else
- {
- igraph_add_edge(&hier->group_graph, group->vertex_id, superior_group->vertex_id);
- group->ref_by_superior_group_cnt++;
- superior_group->ref_by_subordinate_group_cnt++;
- ret=0;
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-}
-
-int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id)
-{
- int ret=0;
- struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
-
- //No hash write operation, LOCK protection is unnecessary.
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(group==NULL)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Del group %d from group %d failed, group %d not exisited.",
- group_id, superior_group_id, group_id);
- return -1;
- }
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group);
- if(superior_group==NULL)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Del group %d from group %d failed, superior group %d not exisited.",
- group_id, superior_group_id, superior_group_id);
- return -1;
- }
- igraph_es_t es;
- igraph_integer_t edge_num_before=0, edge_num_after=0;
-
- edge_num_before=igraph_ecount(&hier->group_graph);
- // The edges between the given pairs of vertices will be included in the edge selection.
- //The vertex pairs must be given as the arguments of the function call, the third argument
- //is the first vertex of the first edge, the fourth argument is the second vertex of the
- //first edge, the fifth is the first vertex of the second edge and so on. The last element
- //of the argument list must be -1 to denote the end of the argument list.
- //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
- ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1);
- assert(ret==IGRAPH_SUCCESS);
- // ignore no such edge to abort().
- igraph_set_error_handler(igraph_error_handler_ignore);
- ret=igraph_delete_edges(&hier->group_graph, es);
- edge_num_after=igraph_ecount(&hier->group_graph);
- igraph_es_destroy(&es);
-
- if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1)
- {
- assert(0);
- return -1;
- }
-
- group->ref_by_superior_group_cnt--;
- superior_group->ref_by_subordinate_group_cnt--;
- return 0;
-
-}
-
-
-int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data)
-{
- //A region rule belongs to ONE group only.
- struct Maat_hierarchy_group* group=NULL;
- struct Maat_hierarchy_region* region=NULL;
- int ret=0;
-
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(!group)
- {
- group=Maat_hierarchy_group_new(hier, group_id);
- }
- HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region);
- if(region)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Add region %d to group %d failed, region already in group %d.",
- region_id,
- group_id,
- region->ref_parent_group->group_id);
- ret=-1;
- }
- else
- {
- region=Maat_hierarchy_region_new(hier, region_id, group_id, table_id, group, user_data);
- ret=0;
- }
- pthread_rwlock_unlock(&hier->rwlock);
-
- return ret;
-}
-void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int region_id)
-{
- struct Maat_hierarchy_region* region=NULL;
- void* ret=NULL;
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region);
- if(region)
- {
- ret=region->user_data;
- region->user_data=NULL;
- }
- pthread_rwlock_unlock(&hier->rwlock);
- return ret;
-}
-int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int group_id, int region_id)
-{
- struct Maat_hierarchy_group* group=NULL;
- struct Maat_hierarchy_region* region=NULL;
- pthread_rwlock_wrlock(&hier->rwlock);
- hier->changed_flag=1;
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
- if(!group)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove region %d from group %d failed, group is not existed.",
- region_id,
- group_id);
- goto error_out;
- }
- HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region);
- if(!region)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Remove region %d from group %d failed, region is not exisited.",
- region_id,
- group_id);
- goto error_out;
- }
-
- assert(region->group_id==group->group_id);
- Maat_hierarchy_region_free(hier, region);
- pthread_rwlock_unlock(&hier->rwlock);
- return 0;
-
-error_out:
- pthread_rwlock_unlock(&hier->rwlock);
- return -1;
-}
-
-static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierarchy* hier)
-{
- struct bool_matcher* bm=NULL;
- size_t compile_num=0, expr_cnt=0;
- struct bool_expr* bool_expr_array=NULL;
- struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
-
- struct Maat_hierarchy_clause_state* clause_state=NULL;
- const struct Maat_hierarchy_clause* clause=NULL;
- size_t i=0, j=0;
- int has_clause_num=0;
- compile_num=HASH_COUNT(hier->hash_compile_by_id);
-
- if(compile_num==0)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "No compile to build.");
- return NULL;
- }
-
- //STEP 1, update clause_id of each compile and literal
- struct Maat_hierarchy_literal_id* literal_ids=NULL;
- size_t n_literal_id=0;
- HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
- {
- has_clause_num=0;
- for(i=0; iclause_states+i;
- clause_state->clause_id=0;
- if(!clause_state->in_use)
- {
- continue;
- }
- has_clause_num++;
- literal_ids=(struct Maat_hierarchy_literal_id*)utarray_eltptr(clause_state->literal_ids, 0);
- n_literal_id=utarray_len(clause_state->literal_ids);
- clause=Maat_hierarchy_clause_fetch(hier, literal_ids, n_literal_id);
- clause_state->clause_id=clause->clause_id;
- }
- assert(has_clause_num==compile->actual_clause_num);
- }
-
- //STEP 2, serial compile clause states to a bool expression array.
- compile_num=HASH_COUNT(hier->hash_compile_by_id);
- bool_expr_array=ALLOC(struct bool_expr, compile_num);
- HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
- {
- for(i=0, j=0; iclause_states[i].in_use)
- {
- if(compile->clause_states[i].not_flag)
- {
- compile->not_clause_cnt++;
- }
- bool_expr_array[expr_cnt].items[j].item_id=compile->clause_states[i].clause_id;
- bool_expr_array[expr_cnt].items[j].not_flag=compile->clause_states[i].not_flag;
- j++;
- }
- }
- //some compile may have zero groups, e.g. default policy.
- if(j==(size_t)compile->declared_clause_num&&j>0)
- {
- bool_expr_array[expr_cnt].expr_id=compile->compile_id;
- bool_expr_array[expr_cnt].user_tag=compile;
- bool_expr_array[expr_cnt].item_num=j;
- expr_cnt++;
- }
- }
-
- //STEP 3, build the bool matcher.
- size_t mem_size=0;
- if(expr_cnt==0)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "No bool expression to build.");
- goto error_out;
- }
- bm=bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
- if(bm!=NULL)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy,
- "Build bool matcher of %zu expressions with %zu bytes memory.",
- expr_cnt,
- mem_size);
- }
- else
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy,
- "Build bool matcher failed!");
- }
-
-error_out:
-
-
- free(bool_expr_array);
- bool_expr_array=NULL;
-
- return bm;
-}
-static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier)
-{
- struct Maat_hierarchy_group* group=NULL, *tmp=NULL;
- struct Maat_hierarchy_group* superior_group=NULL;
- int tmp_vid=0;
- size_t i=0, top_group_cnt=0;
- int* temp_group_ids=NULL;
-
- igraph_bool_t is_dag;
- igraph_is_dag(&(hier->group_graph), &is_dag);
- if(!is_dag)
- {
- MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module,
- "Sub group cycle detected!");
- return -1;
- }
- hier->group_graph_vcount=igraph_vcount(&hier->group_graph);
- igraph_vector_init(&(hier->dfs_vids), hier->group_graph_vcount);
-
- HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp)
- {
- top_group_cnt=0;
- temp_group_ids=NULL;
- //Orphan, Not reference by any one, free it.
- if(group->ref_by_compile_cnt==0
- && group->ref_by_superior_group_cnt==0
- && group->ref_by_subordinate_group_cnt==0
- && group->ref_by_region_cnt==0)
- {
-
- free(group->top_group_ids);
- group->top_group_ids=NULL;
- Maat_hierarchy_group_free(hier, group);
- continue;
- }
-
- //A group is need to build top groups when it has regions and referenced by superior groups or compiles.
- if(group->ref_by_region_cnt>0 && (group->ref_by_compile_cnt>0 || group->ref_by_superior_group_cnt>0))
- {
- if(group->ref_by_superior_group_cnt==0)
- {
- //fast path, group is only referenced by compile rules.
- top_group_cnt=1;
- temp_group_ids=ALLOC(int, top_group_cnt);
- temp_group_ids[0]=group->group_id;
- }
- else
- {
- igraph_vector_t *vids=&(hier->dfs_vids);
- igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT,
- 0, vids, NULL, NULL, NULL, NULL, NULL, NULL);
-
- temp_group_ids=ALLOC(int, effective_vertices_count(vids));
-
- for(i=0; i<(size_t)igraph_vector_size(vids); i++)
- {
- tmp_vid=(int) VECTOR(*vids)[i];
- if(tmp_vid<0)
- {
- break;
- }
- HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group);
- if(superior_group->ref_by_compile_cnt>0)//including itself
- {
- temp_group_ids[top_group_cnt]=superior_group->group_id;
- top_group_cnt++;
- }
- }
- }
-
- }
-
- free(group->top_group_ids);
- group->top_group_cnt=top_group_cnt;
- group->top_group_ids=ALLOC(int, group->top_group_cnt);
- memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt);
-
- free(temp_group_ids);
- temp_group_ids=NULL;
- }
- igraph_vector_destroy(&hier->dfs_vids);
- return 0;
-}
-
-struct region2clause_value* Maat_hierarchy_build_region2clause_hash(struct Maat_hierarchy* hier)
-{
- size_t i=0, j=0, k=0;
- struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL;
- struct Maat_hierarchy_literal_id* literal_id=NULL;
- struct Maat_hierarchy_clause_state* clause_state=NULL;
-
- struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL;
- struct Maat_hierarchy_group* group=NULL;
- struct group2region* g2r_hash=NULL, *g2r=NULL, *g2r_tmp=NULL;
- struct region2clause_value* region2clause_hash=NULL, *r2c_val=NULL;
- struct region2clause_key r2c_key;
-
- //Build a temporary hash that maps group to its regions.
- HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region)
- {
- group=region->ref_parent_group;
- for(i=0; itop_group_cnt; i++)
- {
- HASH_FIND_INT(g2r_hash, group->top_group_ids+i, g2r);
- if(!g2r)
- {
- g2r=ALLOC(struct group2region, 1);
- utarray_new(g2r->region_ids, &ut_region_id_icd);
- utarray_reserve(g2r->region_ids, group->ref_by_region_cnt);
- g2r->group_id=group->top_group_ids[i];
- HASH_ADD_INT(g2r_hash, group_id, g2r);
- }
-
- //One region belongs to one group, one group may have many regions. So duplicate region check is unnecessary.
- //if(utarray_find(g2r->region_ids, &(region->region_id), compare_region_id)) assert(0);
-
- utarray_push_back(g2r->region_ids, &(region->region_id));
- //utarray_sort(g2r->region_ids, compare_region_id);
- }
- }
-
- //Build short cut hash that maps region_id+vt_id to clause_ids.
- HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile)
- {
- for(i=0; iclause_states+i;
- if(!clause_state->in_use)
- {
- continue;
- }
- for(j=0; jliteral_ids); j++)
- {
- literal_id=(struct Maat_hierarchy_literal_id*)utarray_eltptr(clause_state->literal_ids, j);
- HASH_FIND(hh_group_id, hier->hash_group_by_id, &(literal_id->group_id), sizeof(literal_id->group_id), group);
- if(!group)
- {
- continue;
- }
- HASH_FIND_INT(g2r_hash, &(group->group_id), g2r);
- if(!g2r)//group declared by compile, but has no subordinate or region.
- {
- continue;
- }
- for(k=0; kregion_ids); k++)
- {
- r2c_key.region_id=*((int*)utarray_eltptr(g2r->region_ids, k));
- r2c_key.vt_id=literal_id->vt_id;
- HASH_FIND(hh, region2clause_hash, &r2c_key, sizeof(r2c_key), r2c_val);
- if(!r2c_val)
- {
- r2c_val=ALLOC(struct region2clause_value, 1);
- r2c_val->key=r2c_key;
- r2c_val->group_id=g2r->group_id;
- utarray_new(r2c_val->clause_ids, &ut_clause_id_icd);
- HASH_ADD(hh, region2clause_hash, key, sizeof(r2c_val->key), r2c_val);
- }
- if(utarray_find(r2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id))
- {
- continue;
- }
- utarray_push_back(r2c_val->clause_ids, &(clause_state->clause_id));
- utarray_sort(r2c_val->clause_ids, compare_clause_id);
- }
- }
- }
-
- }
- int tmp1=0, tmp2=0;
- HASH_ITER(hh, g2r_hash, g2r, g2r_tmp)
- {
- HASH_DEL(g2r_hash, g2r);
- //Sanity Check
- utarray_sort(g2r->region_ids, compare_region_id);
- for(i=1; iregion_ids); i++)
- {
- tmp1=*((int*)utarray_eltptr(g2r->region_ids, i-1));
- tmp2=*((int*)utarray_eltptr(g2r->region_ids, i));
- assert(tmp1!=tmp2);
- }
- utarray_free(g2r->region_ids);
- g2r->region_ids=NULL;
- free(g2r);
- }
-
- MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy,
- "Build region2clause hash with %llu element.",
- HASH_COUNT(region2clause_hash));
-
- return region2clause_hash;
-}
-int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
-{
- int ret=0;
- struct bool_matcher* new_bm=NULL, *old_bm=NULL;
- struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL;
-
- //Read hier from update thread is OK.
- if(!hier->changed_flag)
- {
- return ret;
- }
- ret=Maat_hierarchy_build_top_groups(hier);
- new_bm=Maat_hierarchy_build_bool_matcher(hier);
- new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier);
-
- pthread_rwlock_wrlock(&hier->rwlock);
-
- old_bm=hier->bm;
- old_region2clause_hash=hier->hash_region2clause;
-
- hier->bm=new_bm;
- hier->hash_region2clause=new_region2clause_hash;
- hier->changed_flag=0;
- pthread_rwlock_unlock(&hier->rwlock);
-
- Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free);
- Maat_garbage_bagging(hier->ref_garbage_bin, old_region2clause_hash, (void (*)(void*))Maat_hierarchy_free_region2clause_hash);
-
- return ret;
-}
-
-
-struct Maat_hierarchy_compile_mid
-{
- int thread_num;
- int Nth_scan;
- time_t hier_ver;
- size_t this_scan_region_hit_cnt;
- int not_clause_hitted_flag;
- int is_no_count_scan;
- size_t hit_path_cnt;
-
- UT_array* _internal_hit_paths;
- UT_array* _all_hit_clause_array;
- UT_array* this_scan_hit_clause_ids;
-};
-
-struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hierarchy* hier, int thread_num)
-{
- struct Maat_hierarchy_compile_mid* mid=ALLOC(struct Maat_hierarchy_compile_mid, 1);
- mid->thread_num=thread_num;
- mid->hier_ver=hier->version;
- utarray_new(mid->_internal_hit_paths, &ut_hit_path_icd);
- utarray_new(mid->_all_hit_clause_array, &ut_clause_id_icd);
- utarray_new(mid->this_scan_hit_clause_ids, &ut_clause_id_icd);
- return mid;
-}
-void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid)
-{
- utarray_free(mid->_internal_hit_paths);
- utarray_free(mid->_all_hit_clause_array);
- utarray_free(mid->this_scan_hit_clause_ids);
- free(mid);
-}
-int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid)
-{
- return mid->not_clause_hitted_flag;
-}
-
-static int Maat_hierarchy_compile_has_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id)
-{
- int i=0;
- struct Maat_hierarchy_literal_id* tmp=NULL;
- struct Maat_hierarchy_clause_state* clause_state=NULL;
- for(i=0; iclause_states+i;
- if(!clause_state->in_use)
- {
- continue;
- }
- tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id);
- if(tmp)
- {
- assert(tmp->group_id==literal_id->group_id && tmp->vt_id==literal_id->vt_id);
- return 1;
- }
- }
- return 0;
-}
-static int Maat_hierarchy_is_hit_path_existed(const struct Maat_hit_path_t* hit_paths, size_t n_path, const struct Maat_hit_path_t* find)
-{
- size_t i=0;
- for(i=0; iexpr_match_buff+mid->thread_num*MAX_SCANNER_HIT_NUM;
- struct Maat_hit_path_t tmp_path;
- if(hier->version!=mid->hier_ver)
- {
- return 0;
- }
- pthread_rwlock_rdlock(&hier->rwlock);
-
- for(i=0; i_internal_hit_paths); i++)
- {
- p=(struct Maat_hierarchy_internal_hit_path*)utarray_eltptr(mid->_internal_hit_paths, i);
-
- HASH_FIND_INT(hier->hash_region_by_id, &(p->region_id), region);
- if(!region)
- {
- continue;
- }
- group=region->ref_parent_group;
- if(group->top_group_cnt==0 && n_made_by_regionNth_scan;
- hit_paths[n_made_by_region].region_id=p->region_id;
- hit_paths[n_made_by_region].sub_group_id=group->group_id;
- hit_paths[n_made_by_region].top_group_id=-1;
- hit_paths[n_made_by_region].virtual_table_id=p->virtual_table_id;
- hit_paths[n_made_by_region].compile_id=-1;
- n_made_by_region++;
- }
- else
- {
- for(j=0; jtop_group_cnt&& n_made_by_regionNth_scan;
- hit_paths[n_made_by_region].region_id=p->region_id;
- hit_paths[n_made_by_region].sub_group_id=group->group_id;
- hit_paths[n_made_by_region].top_group_id=group->top_group_ids[j];
- hit_paths[n_made_by_region].virtual_table_id=p->virtual_table_id;
- hit_paths[n_made_by_region].compile_id=-1;
- }
- }
- }
-
- bool_match_ret=bool_matcher_match(hier->bm,
- (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array),
- expr_match, MAX_SCANNER_HIT_NUM);
- for(i=0; imagic==MAAT_HIER_COMPILE_MAGIC);
- assert((unsigned long long)compile->compile_id==expr_match[i].expr_id);
- if(compile->actual_clause_num==0 || !compile->user_data)
- {
- continue;
- }
- for(j=0; jcompile_id;
- }
- else
- {
- tmp_path=hit_paths[j];
- tmp_path.compile_id=compile->compile_id;
- if(Maat_hierarchy_is_hit_path_existed(hit_paths, n_made_by_region+n_made_by_compile, &tmp_path))
- {
- hit_paths[n_made_by_region+n_made_by_compile]=tmp_path;
- n_made_by_compile++;
- }
- }
- }
- }
- }
-
- pthread_rwlock_unlock(&hier->rwlock);
- return n_made_by_region+n_made_by_compile;
-}
-
-
-void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
-{
- size_t i=0;
- unsigned long long *clause_id=0;
-
- if(mid->Nth_scan!=Nth_scan)
- {
- assert(mid->this_scan_region_hit_cnt==0);
- mid->Nth_scan=Nth_scan;
- utarray_clear(mid->this_scan_hit_clause_ids);
- }
-
- int ret=0;
- ret=Maat_hierarchy_hit_path_add(mid->_internal_hit_paths, region_id, virtual_table_id, Nth_scan, Nth_region_result);
- if(!ret)
- {
- return;
- }
- mid->hit_path_cnt++;
- mid->this_scan_region_hit_cnt++;
-
-
- struct region2clause_value* r2c_val=NULL;
- struct region2clause_key r2c_key;
- r2c_key.region_id=region_id;
- r2c_key.vt_id=virtual_table_id;
- HASH_FIND(hh, hier->hash_region2clause, &r2c_key, sizeof(r2c_key), r2c_val);
- if(!r2c_val)
- {
- return;
- }
- size_t new_clause_idx=utarray_len(mid->this_scan_hit_clause_ids);
- for(i=0; iclause_ids); i++)
- {
- clause_id=(unsigned long long*)utarray_eltptr(r2c_val->clause_ids, i);
- if(utarray_find(mid->_all_hit_clause_array, clause_id, compare_clause_id))
- {
- continue;
- }
- utarray_push_back(mid->this_scan_hit_clause_ids, clause_id);
- }
- if(utarray_len(mid->this_scan_hit_clause_ids)-new_clause_idx)
- {
- utarray_reserve(mid->_all_hit_clause_array, utarray_len(mid->this_scan_hit_clause_ids)-new_clause_idx);
- for(i=new_clause_idx; ithis_scan_hit_clause_ids); i++)
- {
- clause_id=(unsigned long long *)utarray_eltptr(mid->this_scan_hit_clause_ids, i);
- utarray_push_back(mid->_all_hit_clause_array, clause_id);
- }
- utarray_sort(mid->_all_hit_clause_array, compare_clause_id);
- }
- return;
-}
-static int Maat_hierarchy_compile_has_clause(struct Maat_hierarchy_compile* compile, unsigned long long clause_id)
-{
- size_t i=0;
- struct Maat_hierarchy_clause_state* clause_state=NULL;
- for(i=0; iclause_states+i;
- if(!clause_state->in_use)
- {
- continue;
- }
- if(clause_state->clause_id==clause_id)
- {
- return 1;
- }
- }
- return 0;
-}
-
-
-static size_t Maat_hierarchy_compile_mid_if_new_hit_compile(struct Maat_hierarchy_compile_mid* mid, struct Maat_hierarchy_compile* compile)
-{
- size_t r_in_c_cnt=0, i=0;
- int ret=0;
-
- unsigned long long new_hit_clause_id=0;
- for(i=0; ithis_scan_hit_clause_ids); i++)
- {
-
- new_hit_clause_id=*(unsigned long long*)utarray_eltptr(mid->this_scan_hit_clause_ids, i);
- ret=Maat_hierarchy_compile_has_clause(compile, new_hit_clause_id);
- if(ret)
- {
- r_in_c_cnt++;
- }
- }
- return r_in_c_cnt;
-}
-
-
-int Maat_hierarchy_region_compile(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz)
-{
- int bool_match_ret=0, i=0;
- struct Maat_hierarchy_compile* compile=NULL;
- struct bool_expr_match *expr_match=hier->expr_match_buff+mid->thread_num*MAX_SCANNER_HIT_NUM;
-
- size_t r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hit_cnt;
- size_t ud_result_cnt=0;
- if(!hier->bm||0==utarray_len(mid->_all_hit_clause_array)||hier->version!=mid->hier_ver)
- {
- mid->this_scan_region_hit_cnt=0;
- return 0;
- }
- bool_match_ret=bool_matcher_match(hier->bm,
- (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array),
- expr_match, MAX_SCANNER_HIT_NUM);
- for(i=0; imagic==MAAT_HIER_COMPILE_MAGIC);
- assert((unsigned long long)compile->compile_id==expr_match[i].expr_id);
- if(compile->actual_clause_num==0)
- {
- continue;
- }
- r_in_c_cnt=Maat_hierarchy_compile_mid_if_new_hit_compile(mid, compile);
- if(compile->not_clause_cnt>0 && !is_last_compile)
- {
- mid->not_clause_hitted_flag=1;
- }
- else if(compile->user_data)//For compile may be dettached by Maat_hierarchy_compile_dettach_user_data, only return non-NULL userdata.
- {
- if(r_in_c_cnt>0 || //compile hitted becasue of new reigon
- this_scan_region_hits==0) //or hit a compile that refer a NOT-logic group in previous scan.
- {
- user_data_array[ud_result_cnt]=compile->user_data;
- ud_result_cnt++;
- }
- }
- }
-
- mid->this_scan_region_hit_cnt=0;
- return ud_result_cnt;
-}
-
diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp
deleted file mode 100644
index efadc00..0000000
--- a/src/entry/Maat_rule.cpp
+++ /dev/null
@@ -1,2815 +0,0 @@
-#include
-#include
-#include //tolower
-#include
-#include //inet_pton
-#include //inet_pton
-#include //inet_pton
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include
-#include
-#include
-#include
-
-#include "Maat_rule.h"
-#include "Maat_rule_internal.h"
-#include "Maat_utils.h"
-#include "Maat_hierarchy.h"
-#include "Maat_garbage_collection.h"
-
-#include "json2iris.h"
-#include "cJSON.h"
-#include "dynamic_array.h"
-#include "alignment_int64.h"
-#include "config_monitor.h"
-
-#include "map_str2int.h"
-#include "stream_fuzzy_hash.h"
-#include "gram_index_engine.h"
-
-#include "bool_matcher.h"
-
-#define GIT_VERSION_CATTER(v) __attribute__((__used__)) const char * GIT_VERSION_##v = NULL
-#define GIT_VERSION_EXPEND(v) GIT_VERSION_CATTER(v)
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
- /* VERSION TAG */
-#ifdef GIT_VERSION
- GIT_VERSION_EXPEND(GIT_VERSION);
-#else
- static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
-#endif
-#undef GIT_VERSION_CATTER
-#undef GIT_VERSION_EXPEND
-
-#ifdef __cplusplus
-}
-#endif
-
-int MAAT_FRAME_VERSION_3_7_0_20220823=1;
-
-int is_valid_table_name(const char* str)
-{
- size_t i=0, integer_cnt=0;
- for(i=0; i='0'&&str[i]<='9')
- {
- integer_cnt++;
- }
- }
- if(strlen(str)==0 ||
- integer_cnt==strlen(str) ||
- 0==strcasecmp(str, "null"))
- {
- return 0;
- }
- return 1;
-}
-int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
-{
- switch(expr_type)
- {
- case EXPR_TYPE_STRING:
- case EXPR_TYPE_AND:
- case EXPR_TYPE_REGEX:
- case EXPR_TYPE_OFFSET:
- return 1;
- default:
- return 0;
- }
-}
-int is_valid_match_method(enum MAAT_MATCH_METHOD match_method)
-{
- switch(match_method)
- {
- case MATCH_METHOD_SUB:
- case MATCH_METHOD_SUFFIX:
- case MATCH_METHOD_PREFIX:
- case MATCH_METHOD_COMPLETE:
- return 1;
- default:
- return 0;
- }
-}
-
-
-iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
-{
- const char *from_s=charset_get_name(from);
- const char *to_s=charset_get_name(to);
- iconv_t cd;
- if(from==CHARSET_GBK&&to==CHARSET_BIG5)
- {
- from_s="gb2312";
- }
- if(from>=MAX_CHARSET_NUM||to>=MAX_CHARSET_NUM)
- {
- return (iconv_t)-1;
- }
-
- if(scanner->iconv_handle[to][from]==NULL)
- {
- scanner->iconv_handle[to][from]=iconv_open(to_s, from_s);
- }
- cd=scanner->iconv_handle[to][from];
- return cd;
-}
-
-int iconv_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
-{
- size_t ret;
- int copy_len=0;
- char* copy_buf=NULL;
- if(srclen==0||src==NULL)
- {
- return -1;
- }
- iconv_t cd=maat_iconv_open(scanner,to, from);
- if(cd!=(iconv_t)-1)
- {
- char * pInBuff=src;
- size_t iInBuffLen=srclen;
-
- size_t iOutBuffLen=10*iInBuffLen;
- char * pOutBuff=(char *)malloc(iOutBuffLen);
-
- char * pLeftBuff=pOutBuff;
- size_t iLeftLen=iOutBuffLen;
-
- ret=iconv(cd, &pInBuff, &iInBuffLen, &pLeftBuff, &iLeftLen);
- if(ret!=(size_t)(-1))
- {
-
- if(to==CHARSET_UNICODE&&
- (*(unsigned short*)pOutBuff==0xFFFE||*(unsigned short*)pOutBuff==0XFEFF))//jump unicode 2 bytes BOM, 0xFF 0xFE
- {
- copy_len=iOutBuffLen-iLeftLen-2;
- copy_buf=pOutBuff+2;
- }
- else
- {
- copy_len=iOutBuffLen-iLeftLen;
- copy_buf=pOutBuff;
- }
- assert(copy_len<=*dstlen);
- *dstlen=copy_len;
- memcpy(dst,copy_buf,*dstlen);
-
- free(pOutBuff);
- return 1;
- }
- else
- {
- free(pOutBuff);
- return -1;
- }
- }
- else
- {
- return -1;
- }
-
-}
-int URLEncode(const char* str, const int strSize, char* result, const int resultSize)
-{
- int i;
- int j = 0;//for result index
- char ch;
-
- if ((str==NULL) || (result==NULL) || (strSize<=0) || (resultSize<=0))
- {
- return -1;
- }
-
- for ( i=0; (i='A') && (ch<='Z')) ||
- ((ch>='a') && (ch<='z')) ||
- ((ch>='0') && (ch<='9')))
- {
- result[j++] = ch;
- }
- else if (ch == ' ')
- {
- result[j++] = '+';
- }
- else if (ch == '.' || ch == '-' || ch == '_' || ch == '*')
- {
- result[j++] = ch;
- }
- else
- {
- if (j+3 < resultSize)
- {
- sprintf(result+j, "%%%02X", (unsigned char)ch);
- j += 3;
- }
- else
- {
- return -1;
- }
- }
- }
-
- result[j] = '\0';
- return j;
-}
-int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, const int dstsize)
-{
- int i=0,j=0;
- assert(srclen%2==0);//unicode must be 2 bytes aligned.
- while(i0;p[i]=p[i]/2)
- {
- bits_cnt++;
- }
- }
- return bits_cnt;
-}
-//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]}
-int parse_accept_tag(const char* value, struct rule_tag** result, void* logger)
-{
- cJSON* json=NULL, *array=NULL,*tag=NULL, *tmp=NULL;
- struct rule_tag* p=NULL;
- int n_tags=0;
- json=cJSON_Parse(value);
- if(!json)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
- "MAAT_OPT_ACCEPT_TAGS Error before: %-200.200s",cJSON_GetErrorPtr());
- return 0;
- }
- array=cJSON_GetObjectItem(json, "tags");
- n_tags=cJSON_GetArraySize(array);
- p=ALLOC(struct rule_tag, n_tags);
- for(int i=0;ivaluestring);
- tmp=cJSON_GetObjectItem(tag, "value");
- p[i].tag_val=_maat_strdup(tmp->valuestring);
- }
- cJSON_Delete(json);
- *result=p;
- return n_tags;
-}
-static int compare_each_tag(cJSON* tag_obj, const struct rule_tag* accept_tags, int n_accept)
-{
- const char* tag_name;
- const char* tag_val;
- int n_val;
- cJSON *tab_name_obj=NULL, *tag_vals_array=NULL, *tag_val_obj=NULL;
-
- int i=0, j=0, name_matched=0;
- tab_name_obj=cJSON_GetObjectItem(tag_obj,"tag");
- if(!tab_name_obj||tab_name_obj->type!=cJSON_String)
- {
- goto error_out;
- }
- tag_name=tab_name_obj->valuestring;
- tag_vals_array=cJSON_GetObjectItem(tag_obj,"value");
- if(!tag_vals_array||tag_vals_array->type!=cJSON_Array)
- {
- goto error_out;
- }
- n_val=cJSON_GetArraySize(tag_vals_array);
- for(i=0;itype!=cJSON_String)
- {
- goto error_out;
- }
- tag_val=tag_val_obj->valuestring;
- // compare a/b/c with a/b/c/d is a miss.
- if(strlen(accept_tags[i].tag_val)0)
- {
- return 0;
- }
- else
- {
- return 1;
- }
-error_out:
- return -1;
-
-}
-//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}]
-static int compare_each_tag_set(cJSON* tag_set, const struct rule_tag* accept_tags, int n_accept)
-{
- cJSON *tag_obj=NULL;
- int n_tag=0, ret=0, matched=0;
- n_tag=cJSON_GetArraySize(tag_set);
- for(int i=0; itype!=cJSON_Object)
- {
- goto error_out;
- }
- ret=compare_each_tag(tag_obj, accept_tags, n_accept);
- if(ret<0)
- {
- return -1;
- }
- if(ret==1)
- {
- matched++;
- }
- }
- if(matched==n_tag)
- {
- return 1;
- }
- else
- {
- return 0;
- }
-error_out:
- return -1;
-}
-//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]}
-//@return 1 on match, 0 on not match, -1 on error.
-static int compare_accept_tag(const char* value,const struct rule_tag* accept_tags, int n_tags)
-{
- cJSON *json=NULL;
- cJSON *tag_set_array=NULL, *tag_set=NULL;
- int ret=-1, n_set=0;
- json=cJSON_Parse(value);
- if(!json)
- {
- goto error_out;
- }
- tag_set_array=cJSON_GetObjectItem(json, "tag_sets");
- if(!tag_set_array||tag_set_array->type!=cJSON_Array)
- {
- goto error_out;
- }
- n_set=cJSON_GetArraySize(tag_set_array);
- for(int i=0; itype!=cJSON_Array)
- {
- goto error_out;
- }
- ret=compare_each_tag_set(tag_set, accept_tags, n_tags);
- if(ret!=0)//match or error occurs.
- {
- break;
- }
- }
-error_out:
- cJSON_Delete(json);
- return ret;
-}
-
-MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc)
-{
- MAAT_RULE_EX_DATA ad=NULL;
- struct Maat_rule_t rule;
- fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
- ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp);
- return ad;
-}
-void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc)
-{
- struct Maat_rule_t rule;
- fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
- ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp);
- return;
-}
-
-void EMPTY_FREE(void*p)
-{
- return;
-}
-
-struct Maat_compile_rule* create_compile_rule(struct Maat_rule_head* p_head, const char* service_define, int declared_grp_num, double exec_seq, const struct Maat_table_schema* table)
-{
- int i=0;
- struct Maat_compile_rule*p=ALLOC(struct Maat_compile_rule, 1);
- p->magic_num=COMPILE_RULE_MAGIC;
- p->head=*p_head;
- p->declared_clause_num=declared_grp_num;
- p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
-
- //protect by feather->background_update_mutex
- p->ref_table=table;
- p->head.serv_def_len=strlen(service_define)+1;
- p->service_defined=ALLOC(char, p->head.serv_def_len);
- memcpy(p->service_defined, service_define, p->head.serv_def_len);
- p->evaluation_order=exec_seq;
-
- for(i=0; icompile.ex_data_num; i++)
- {
- p->ads[i]=rule_ex_data_new(&p->head, p->service_defined, table->compile.ex_desc+i);
- }
- p->is_valid=1;
- p->compile_id=p_head->config_id;
- pthread_rwlock_init(&p->rwlock, NULL);
- return p;
-}
-
-void destroy_compile_rule(struct Maat_compile_rule* compile_rule)
-{
- int i=0;
- const struct compile_table_schema* compile_desc= &(compile_rule->ref_table->compile);
- assert(compile_rule->magic_num==COMPILE_RULE_MAGIC);
- for(i=0; iex_data_num; i++)
- {
- rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i);
- compile_rule->ads[i]=NULL;
- }
- free(compile_rule->ads);
-
- compile_rule->is_valid=0;
- compile_rule->declared_clause_num=-1;
- free(compile_rule->service_defined);
- compile_rule->service_defined=NULL;
- free(compile_rule);
- return;
-}
-
-
-scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset)
-{
- scan_rule_t* p_rule=(scan_rule_t* )calloc(sizeof(scan_rule_t),1);
- p_rule->rule_type=RULETYPE_STR;
- p_rule->sub_type=sub_type;
-
- p_rule->string_rule.case_sensitive=is_case_sensitive;
- p_rule->string_rule.match_mode=0;
- p_rule->string_rule.l_offset=-1;
- p_rule->string_rule.r_offset=-1;
- switch(match_method)
- {
- case MATCH_METHOD_COMPLETE:
- p_rule->string_rule.match_mode=1;
- break;
- case MATCH_METHOD_PREFIX:
- p_rule->string_rule.l_offset=-2;
- break;
- case MATCH_METHOD_SUFFIX:
- p_rule->string_rule.r_offset=-2;
- break;
- case MATCH_METHOD_SUB:
- p_rule->string_rule.l_offset=l_offset;
- p_rule->string_rule.r_offset=r_offset;
- break;
- default:
- assert(0);
- break;
- }
- p_rule->string_rule.len=len;
- p_rule->string_rule.str=(char*)calloc(sizeof(char),len);
- memcpy(p_rule->string_rule.str,string,len);
- return p_rule;
-
-}
-void destroy_rs_str_rule(scan_rule_t* p_rule)
-{
- free(p_rule->string_rule.str);
- free(p_rule);
-}
-scan_rule_t* create_rs_ip_rule(unsigned int sub_type,struct db_ip_rule_t *db_ip_rule)
-{
- scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1);
- if(db_ip_rule->addr_type==4)
- {
- p_rule->rule_type=RULETYPE_IPv4;
- memcpy(&(p_rule->ipv4_rule),&(db_ip_rule->ipv4_rule),sizeof(p_rule->ipv4_rule));
- }
- else
- {
- p_rule->rule_type=RULETYPE_IPv6;
- memcpy(&(p_rule->ipv6_rule),&(db_ip_rule->ipv6_rule),sizeof(p_rule->ipv6_rule));
- }
- p_rule->sub_type=sub_type;
- return p_rule;
-}
-void destroy_rs_ip_rule(scan_rule_t* p)
-{
- free(p);
-}
-scan_rule_t* create_rs_intval_rule(unsigned int sub_type,struct db_interval_rule *intval_rule)
-{
- scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1);
- p_rule->rule_type=RULETYPE_INT;
- p_rule->sub_type=sub_type;
- p_rule->interval_rule.lb=intval_rule->intval.lb;
- p_rule->interval_rule.ub=intval_rule->intval.ub;
- return p_rule;
-}
-void destroy_rs_intval_rule(scan_rule_t* p)
-{
- free(p);
-}
-
-struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para,int table_id)
-{
- struct op_expr_t* op_expr=NULL;
- op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1);
- op_expr->no_effect_convert_cnt=0;
- op_expr->convert_failed=0;
- op_expr->p_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),1);
- op_expr->p_expr->expr_id=expr_id;
- op_expr->p_expr->operation=operation;
- op_expr->p_expr->rnum=0;
- op_expr->p_expr->rules=NULL;
- op_expr->p_expr->tag=u_para;
- op_expr->table_id=table_id;
- return op_expr;
-}
-void destroy_op_expr(struct op_expr_t* op_expr)
-{
- unsigned int i=0;
- for(i=0;ip_expr->rnum;i++)
- {
- switch(op_expr->p_rules[i]->rule_type)
- {
- case RULETYPE_STR:
- case RULETYPE_REG:
- destroy_rs_str_rule(op_expr->p_rules[i]);
- break;
- case RULETYPE_IPv4:
- case RULETYPE_IPv6:
- destroy_rs_ip_rule(op_expr->p_rules[i]);
- break;
- case RULETYPE_INT:
- destroy_rs_intval_rule(op_expr->p_rules[i]);
- break;
- default:
- assert(0);
- break;
- }
- op_expr->p_rules[i]=NULL;
- }
- free(op_expr->p_expr);
- op_expr->p_expr=NULL;
- free(op_expr);
-}
-void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
-{
- int idx=op_expr->p_expr->rnum;
- op_expr->p_rules[idx]=p_rule;
- op_expr->p_expr->rnum++;
- op_expr->rule_type=p_rule->rule_type;
- return;
-}
-void Maat_region_inner_free(struct Maat_region_inner* region)
-{
- assert(region->magic_num==REGION_RULE_MAGIC);
- assert(region->expr_id_cnt==0||region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb+1);
- region->magic_num=0;
- free(region);
-}
-
-void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region)
-{
- assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb+1);
- region->expr_id_ub--;
- region->expr_id_cnt--;
- return;
-}
-
-struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *feather)
-{
- int scan_thread_num=feather->scan_thread_num;
- UNUSED int ret=0;
-
- struct Maat_scanner* scanner=NULL;
- scanner=ALLOC(struct Maat_scanner, 1);
- scanner->hier=Maat_hierarchy_new(scan_thread_num, feather->logger, feather->garbage_bin);
- Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, (void (*)(void*))destroy_compile_rule);
- Maat_hierarchy_set_region_user_data_free_func(scanner->hier, (void (*)(void*))Maat_region_inner_free);
-
- scanner->district_map=maat_kv_store_new();
-
- scanner->version=version;
- scanner->cfg_num=0;
- scanner->dedup_expr_num=0;
- scanner->max_thread_num=scan_thread_num;
- //optimized for CPU cache_alignment 64
- scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num);
- scanner->region_update_q=MESA_lqueue_create(0, 0);
- scanner->region=rulescan_initialize(scan_thread_num);
-
- //For best scan performance:
- //1.Do NOT set this option, rulescan return no hit detail as default;
- //2.Set necessary STR rule to QUICK;
- if(feather->rule_scan_type==1)
- {
- rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0);
- }
- else if(feather->rule_scan_type==2)
- {
- rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0);
- rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0);
- }
- scanner->ref_garbage_bin=feather->garbage_bin;
- scanner->logger_ref=feather->logger;
- scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
- scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
- scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num, feather->garbage_bin);
- scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr);
- return scanner;
-}
-
-
-
-void destroy_maat_scanner(struct Maat_scanner*scanner)
-{
- long q_cnt=0,data_size=0;
- int i=0,j=0;
- UNUSED int q_ret=0;
- struct op_expr_t* op_expr=NULL;
- if(scanner==NULL)
- {
- return;
- }
- rulescan_destroy(scanner->region);
-
- maat_kv_store_free(scanner->district_map);
- scanner->district_map=NULL;
- assert(scanner->tmp_district_map==NULL);
-
- q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
- for(i=0;iregion_update_q,&op_expr,&data_size);
- assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
- destroy_op_expr(op_expr);
- }
- MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL);
- free(scanner->region_rslt_buff);
- scanner->region_rslt_buff=NULL;
- free(scanner->gie_rslt_buff);
- scanner->gie_rslt_buff=NULL;
- alignment_int64_array_free(scanner->ref_cnt);
- scanner->ref_cnt=NULL;
- for(i=0;iiconv_handle[i][j]!=NULL)
- {
- iconv_close(scanner->iconv_handle[i][j]);
- }
- }
- }
- Maat_table_rt_manager_destroy(scanner->table_rt_mgr);
- scanner->table_rt_mgr=NULL;
- Maat_hierarchy_free(scanner->hier);
- scanner->hier=NULL;
- free(scanner);
- return;
-}
-
-
-unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge)
-{
- unsigned int sub_type=0;
- if(do_charset_merge==TRUE)
- {
- sub_type=table_id<<4|CHARSET_NONE;
- }
- else
- {
- sub_type=table_id<<4|charset;
- }
- assert(sub_typerules);
- free(p);
- return;
-}
-struct _region_stat_t
-{
- int cfg_num;
- union
- {
- int expr_rule_cnt; //expr_type=0,1,3
- int ipv4_rule_cnt;
- };
- union
- {
- int regex_rule_cnt; //expr_type=2
- int ipv6_rule_cnt;
- };
-};
-void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size)
-{
- int op=0;
- if(op_expr->p_expr->operation==0)//add
- {
- op=1;
- }
- else if(op_expr->p_expr->operation==1)//delete
- {
- op=-1;
- }
- else
- {
- assert(0);
- }
- region_stat[op_expr->table_id].cfg_num+=op;
- switch(op_expr->rule_type)
- {
- case RULETYPE_STR:
- region_stat[op_expr->table_id].expr_rule_cnt+=op;
- break;
- case RULETYPE_REG:
- region_stat[op_expr->table_id].regex_rule_cnt+=op;
- break;
- case RULETYPE_INT:
- break;
- case RULETYPE_IPv4:
- region_stat[op_expr->table_id].ipv4_rule_cnt+=op;
- break;
- case RULETYPE_IPv6:
- region_stat[op_expr->table_id].ipv6_rule_cnt+=op;
- break;
- default:
- assert(0);
- break;
- }
- return;
-}
-
-void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner)
-{
- long data_size=0, i=0;
- unsigned int j=0;
- int ret=0;
- unsigned int failed_ids[MAX_FAILED_NUM];
- char failed_info[512], *p=NULL;
- UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
- memset(failed_ids,0,sizeof(failed_ids));
- memset(failed_info,0,sizeof(failed_info));
- const long q_cnt=MESA_lqueue_get_count(expr_queue);
- struct timespec start,end;
- unsigned long long update_interval=0;
- size_t max_table_num=maat_scanner->max_table_num;
- struct _region_stat_t region_counter[max_table_num];
- memset(region_counter, 0, sizeof(region_counter));
- struct Maat_table_runtime* table_rt=NULL;
- if(q_cnt==0)
- {
- return;
- }
- boolean_expr_t* to_update_expr= ALLOC(boolean_expr_t, q_cnt);
- struct op_expr_t* op_expr=NULL;
- for(i=0;ip_expr,sizeof(boolean_expr_t));
- //make a whole memory chunk
- to_update_expr[i].rules= ALLOC(scan_rule_t, op_expr->p_expr->rnum);
- for(j=0;jp_expr->rnum;j++)
- {
- memcpy(&(to_update_expr[i].rules[j]),op_expr->p_rules[j],sizeof(scan_rule_t));
- if(to_update_expr[i].rules[j].rule_type==RULETYPE_REG||to_update_expr[i].rules[j].rule_type==RULETYPE_STR)
- {
- to_update_expr[i].rules[j].string_rule.str=(char*)calloc(sizeof(char),to_update_expr[i].rules[j].string_rule.len);
- memcpy(to_update_expr[i].rules[j].string_rule.str
- ,op_expr->p_rules[j]->string_rule.str
- ,to_update_expr[i].rules[j].string_rule.len);
- }
- }
-
- count_rs_region(op_expr, region_counter, max_table_num);
- destroy_op_expr(op_expr);
- op_expr=NULL;
- }
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module ,
- "rs_handle %p rulescan_update %ld rules.", rs_handle, q_cnt);
- clock_gettime(CLOCK_MONOTONIC, &start);
- ret=rulescan_update(rs_handle, to_update_expr,q_cnt, failed_ids, MAX_FAILED_NUM);
- clock_gettime(CLOCK_MONOTONIC, &end);
- if(ret!=1)
- {
- p=failed_info;
- for(i=0;i10;i++)
- {
- p+=snprintf(p,sizeof(failed_info)-(p-failed_info),"%d,",failed_ids[i+1]);
- }
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "rulescan_update error,when batch update %ld rules,regex error %u.",q_cnt,failed_ids[0]);
- assert(0);
- }
- update_interval=(end.tv_sec-start.tv_sec)*1000000000+end.tv_nsec-start.tv_nsec;
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "rs_handle %p rulescan_update with %2.2e (%llu) ns."
- ,rs_handle
- ,(double)update_interval
- ,update_interval);
- //update scanner's region cnt;
- for(i=0; (size_t)itable_rt_mgr, i);
- if(table_rt==NULL)
- {
- continue;
- }
- switch(table_rt->table_type)
- {
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt;
- table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt;
- assert(table_rt->expr.expr_rule_cnt>=0);
- assert(table_rt->expr.regex_rule_cnt>=0);
- break;
- case TABLE_TYPE_IP:
- case TABLE_TYPE_IP_PLUS:
- table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt;
- table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt;
- break;
- default:
- break;
- }
- assert(table_rt->origin_rule_num>=0);
- }
- for(i=0;idistrict_map, district_str, &district_id);
- if(map_ret<0)
- {
- if(scanner->tmp_district_map==NULL)
- {
- scanner->tmp_district_map=maat_kv_store_duplicate(scanner->district_map);
- }
- map_ret=maat_kv_read(scanner->tmp_district_map, district_str, &district_id);
- if(map_ret<0)
- {
- district_id=scanner->district_num;
- maat_kv_register(scanner->tmp_district_map, district_str, district_id);
- scanner->district_num++;
- }
- }
- return district_id;
-}
-struct Maat_region_inner* Maat_region_inner_new(int group_id, int region_id, int table_id, int district_id)
-{
- struct Maat_region_inner* region=ALLOC(struct Maat_region_inner, 1);
- region->magic_num=REGION_RULE_MAGIC;
- region->region_id=region_id;
- region->group_id=group_id;
- region->table_id=table_id;
- region->district_id=district_id;
- return region;
-}
-void Maat_region_inner_add_expr_id(struct Maat_region_inner* region, int expr_id)
-{
- if(region->expr_id_cnt==0)
- {
- region->expr_id_lb=region->expr_id_ub=expr_id;
- }
- else
- {
- assert(region->expr_id_ub+1==expr_id);
- region->expr_id_ub=expr_id;
- }
- region->expr_id_cnt++;
- return;
-}
-
-int add_expr_ng_rule(struct Maat_table_schema* table, struct db_expr_rule_t* db_rule, struct Maat_scanner *scanner, void* logger)
-{
- int district_id=-1;
- if(table->table_type==TABLE_TYPE_EXPR_PLUS)
- {
- assert(strlen(db_rule->district)>0);
- str_unescape(db_rule->district);
- district_id=get_district_id(scanner, db_rule->district);
- }
- struct hs_expression *hs_expr=ALLOC(struct hs_expression, 1);
- struct hs_pattern *sub_pattern=NULL;
- int i=0, j=0, ret=0;
- int sub_expr_cnt=0;
- char *p=NULL, *saveptr=NULL, *region_string=NULL;
- char *tmp=NULL, *tmp_pattern=NULL;
- hs_expr->type=db_rule->expr_type;
- struct Maat_region_inner* user_tag=NULL;
-
- switch(db_rule->expr_type)
- {
- case EXPR_TYPE_AND:
- case EXPR_TYPE_OFFSET:
- for(i=0, p=db_rule->keywords; ; i++,p=NULL)
- {
- tmp=strtok_r_esc(p, '&', &saveptr);
- if(tmp==NULL)
- {
- break;
- }
- if(i>=MAAT_MAX_EXPR_ITEM_NUM)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id);
- goto error_out;
- }
- if(strlen(tmp)==0) continue;
- sub_pattern=hs_expr->sub_patterns[hs_expr->n_sub_pattern];
- tmp_pattern=tmp;
- if(db_rule->expr_type==EXPR_TYPE_OFFSET)
- {
- sscanf(tmp, "%d-%d:", &(sub_pattern->start_offset),&(sub_pattern->end_offset));
- tmp_pattern=(char*)memchr(tmp_pattern, ':', strlen(tmp_pattern));
- if(tmp_pattern==NULL)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d invalid offset keyword format.", table->table_name[table->updating_name], db_rule->region_id);
- goto error_out;
- }
- tmp_pattern++;//skip ':'
- }
- else
- {
- sub_pattern->start_offset=sub_pattern->end_offset=-1;
- }
- sub_pattern->pattern=_maat_strdup(tmp_pattern);
- str_unescape(hs_expr->sub_patterns[i].pattern);
- hs_expr->n_sub_pattern++;
- }
- break;
- case EXPR_TYPE_REGEX:
- case EXPR_TYPE_STRING:
- hs_expr->n_sub_pattern=1;
- switch(db_rule->match_method)
- {
- case MATCH_METHOD_COMPLETE:
- hs_expr->sub_patterns[0].start_offset=-2;
- hs_expr->sub_patterns[0].end_offset=-2;
- break;
- case MATCH_METHOD_PREFIX:
- hs_expr->sub_patterns[0].start_offset=-2;
- hs_expr->sub_patterns[0].end_offset=-1;
- break;
- case MATCH_METHOD_SUFFIX:
- hs_expr->sub_patterns[0].start_offset=-1;
- hs_expr->sub_patterns[0].end_offset=-2;
- break;
- case MATCH_METHOD_SUB:
- hs_expr->sub_patterns[0].start_offset=-1;
- hs_expr->sub_patterns[0].end_offset=-1;
- break;
- default:
- assert(0);
- break;
- }
- hs_expr->sub_patterns[0].pattern=_maat_strdup(tmp);
- str_unescape(hs_expr->sub_patterns[0].pattern);
- break;
- default:
- break;
- }
- user_tag=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, district_id);
- ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, user_tag);
- if(ret!=0)
- {
- goto error_out;
- }
- hs_expr->user_tag=user_tag;
- for(i=0; in_sub_pattern; i++)
- {
- if(db_rule->is_hexbin)
- {
- hex2bin(hs_expr->sub_patterns[i].pattern, int hex_len, char * binary, int size)
- }
- }
-error_out:
-
- free(hs_expr);
- if(user_tag) Maat_region_inner_free(user_tag);
- return -1;
-}
-int add_expr_rule(struct Maat_table_schema* table,struct db_expr_rule_t* db_rule,struct Maat_scanner *scanner,void* logger)
-{
- unsigned int i=0,j=0;
- char* p=NULL,*saveptr=NULL,*region_string=NULL;
- int region_str_len=0,ret=0,k=0;
- int expr_id=0,district_id=-1;
- struct string_table_schema* expr_desc=&(table->expr);
- scan_rule_t*p_rule=NULL;
-
- enum MAAT_CHARSET dst_charset=CHARSET_NONE;
- char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM], *tmp=NULL;
- int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1};
- for(i=0;itable_type==TABLE_TYPE_EXPR_PLUS)
- {
- assert(strlen(db_rule->district)>0);
- str_unescape(db_rule->district);
- district_id=get_district_id(scanner, db_rule->district);
- }
-
-
- switch(db_rule->expr_type)
- {
- case EXPR_TYPE_AND:
- case EXPR_TYPE_REGEX:
- for(i=0,p=db_rule->keywords;;i++,p=NULL)
- {
- tmp=strtok_r_esc(p, '&', &saveptr);
- if(tmp==NULL)
- {
- break;
- }
- if(i>=MAAT_MAX_EXPR_ITEM_NUM)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id);
- return -1;
- }
- sub_key_array[i]=tmp;
- if(db_rule->expr_type==EXPR_TYPE_REGEX)
- {
- sub_key_array[i]=str_unescape_and(sub_key_array[i]);//regex should use str_unescape_and
- }
- else
- {
- sub_key_array[i]=str_unescape(sub_key_array[i]);
- }
- }
- sub_expr_cnt=i;
- break;
- case EXPR_TYPE_OFFSET:
- for(i=0,p=db_rule->keywords;;i++,p=NULL)
- {
- tmp=strtok_r_esc(p, '&', &saveptr);
- if(tmp==NULL)
- {
- break;
- }
- if(i>=MAAT_MAX_EXPR_ITEM_NUM)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id);
- return -1;
- }
- sub_key_array[i]=tmp;
- sscanf(sub_key_array[i], "%d-%d:", &(key_left_offset[i]),&(key_right_offset[i]));
- if(!(key_left_offset[i]>=0&&key_right_offset[i]>0&&key_left_offset[i]<=key_right_offset[i]))
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d invalid offset.", table->table_name[table->updating_name], db_rule->region_id);
- return -1;
- }
- sub_key_array[i]=(char*)memchr(sub_key_array[i], ':', strlen(sub_key_array[i]));
- if(sub_key_array[i]==NULL)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "Table %s region cfg %d invalid offset keyword format.", table->table_name[table->updating_name], db_rule->region_id);
- return -1;
- }
- sub_key_array[i]++;//jump over ':'
- sub_key_array[i]=str_unescape(sub_key_array[i]);
- }
- sub_expr_cnt=i;
- break;
- case EXPR_TYPE_STRING:
- sub_expr_cnt=1;
- sub_key_array[0]=db_rule->keywords;
- sub_key_array[0]=str_unescape(sub_key_array[0]);
- break;
- default:
- break;
- }
- for(k=0;ktable_name[table->updating_name],db_rule->region_id);
- //this sub string will be skipped before iconv_convert
- }
- }
-
- u_para=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, district_id);
- ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, u_para);
- if(ret!=0)
- {
- Maat_region_inner_free(u_para);
- u_para=NULL;
- return -1;
- }
-
- if(db_rule->is_hexbin==FALSE && db_rule->expr_type!=EXPR_TYPE_REGEX)
- {
- for(j=0;jdst_charset[j];
- if(dst_charset==CHARSET_NONE)
- {
- break;
- }
- expr_id=scanner->exprid_generator++;
- Maat_region_inner_add_expr_id(u_para, expr_id);
- op_expr=create_op_expr(expr_id,
- RULESCAN_OP_ADD,
- u_para,
- table->table_id);
-
- for(k=0;ksrc_charset!=dst_charset)//need convert
- {
-
- ret=universal_charset_convert(scanner,expr_desc->src_charset, dst_charset,
- sub_key_array[k],strlen(sub_key_array[k]),
- region_string, ®ion_str_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module ,
- "Table %s region cfg %d charset convert from %s to %s failed.",
- table->table_name,
- db_rule->region_id,
- charset_get_name(expr_desc->src_charset),
- charset_get_name(dst_charset));
- free(region_string);
- op_expr->convert_failed++;
- expr_desc->iconv_err_cnt++;
- break;
- }
- if(region_str_len==(int)strlen(sub_key_array[k])&&
- 0==memcmp(sub_key_array[k],region_string,region_str_len))
- {
- op_expr->no_effect_convert_cnt++;
- }
- }
- else
- {
- memcpy(region_string,sub_key_array[k],strlen(sub_key_array[k]));
- region_str_len=strlen(sub_key_array[k]);
- }
- p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge)
- ,db_rule->match_method
- ,db_rule->is_case_sensitive
- ,region_string
- ,region_str_len
- ,key_left_offset[k]
- ,key_right_offset[k]);
- op_expr_add_rule(op_expr, p_rule);
- free(region_string);
- region_string=NULL;
- }
- //if each sub string's convert take no effect and src charset is one of the dst.
- //if any sub expr convert failed
- if((TRUE==expr_desc->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)||
- op_expr->convert_failed>0)
- {
- scanner->dedup_expr_num++;
- Maat_region_inner_cancel_last_expr_id(u_para);
- destroy_op_expr(op_expr);
- //redeem expr_id
- scanner->exprid_generator--;
- op_expr=NULL;
- }
- else
- {
- MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*));
- }
- }
-
- }
- else //For hexbin and regex, no need to do charset conversion.
- {
- expr_id=scanner->exprid_generator++;
- Maat_region_inner_add_expr_id(u_para, expr_id);
- op_expr=create_op_expr(expr_id,
- 0, //add
- u_para,
- table->table_id
- );
- for(k=0;kexpr_type==EXPR_TYPE_REGEX)
- {
- p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge),
- db_rule->match_method,
- db_rule->is_case_sensitive,
- sub_key_array[k],
- strlen(sub_key_array[k]),
- key_left_offset[k],
- key_right_offset[k]);
- p_rule->rule_type=RULETYPE_REG;
- }
- else
- {
- region_str_len=strlen(sub_key_array[k])+1;
- region_string=ALLOC(char, region_str_len);
- region_str_len=hex2bin(sub_key_array[k], strlen(sub_key_array[k]), region_string, region_str_len);
-
- p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge),
- db_rule->match_method,
- db_rule->is_case_sensitive,
- region_string,
- region_str_len,
- key_left_offset[k],
- key_right_offset[k]);
-
- free(region_string);
- region_string=NULL;
- }
- op_expr_add_rule(op_expr, p_rule);
- }
- MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*));
- }
- return 0;
-}
-int add_ip_rule(struct Maat_table_schema* table, struct db_ip_rule_t* db_ip_rule, struct Maat_scanner *scanner, void* logger)
-{
- scan_rule_t* p_rule=NULL;
- struct op_expr_t* op_expr=NULL;
- struct Maat_region_inner* u_para=NULL;
- int expr_id=0, ret=0;
-
- u_para=Maat_region_inner_new(db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, -1);
- ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, u_para);
- if(ret!=0)
- {
- Maat_region_inner_free(u_para);
- u_para=NULL;
- return -1;
- }
-
- expr_id=scanner->exprid_generator++;
- Maat_region_inner_add_expr_id(u_para, expr_id);
- op_expr=create_op_expr(expr_id,
- RULESCAN_OP_ADD,
- u_para,
- table->table_id);
- p_rule=create_rs_ip_rule(make_sub_type(table->table_id,CHARSET_NONE,0)
- ,db_ip_rule);
- op_expr_add_rule(op_expr, p_rule);
- MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
- return 0;
-}
-int add_interval_rule(struct Maat_table_schema* table,struct db_interval_rule* intval_rule,struct Maat_scanner *scanner,void* logger)
-{
- scan_rule_t* p_rule=NULL;
- struct op_expr_t* op_expr=NULL;
- struct Maat_region_inner* u_para=NULL;
- int expr_id=0, ret=0,district_id=-1;
-
- if(table->table_type==TABLE_TYPE_INTERVAL_PLUS)
- {
- assert(strlen(intval_rule->district)>0);
- str_unescape(intval_rule->district);
- district_id=get_district_id(scanner, intval_rule->district);
- }
-
- u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, district_id);
- ret=Maat_hierarchy_add_region_to_group(scanner->hier, intval_rule->group_id, intval_rule->region_id, table->table_id, u_para);
- if(ret!=0)
- {
- Maat_region_inner_free(u_para);
- u_para=NULL;
- return -1;
- }
-
- expr_id=scanner->exprid_generator++;
- Maat_region_inner_add_expr_id(u_para, expr_id);
-
- op_expr=create_op_expr(expr_id,
- RULESCAN_OP_ADD,
- u_para,
- table->table_id);
-
- p_rule=create_rs_intval_rule(make_sub_type(table->table_id,CHARSET_NONE,0)
- ,intval_rule);
- op_expr_add_rule(op_expr,p_rule);
- MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
- return 0;
-}
-int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger)
-{
- struct Maat_region_inner* u_para=NULL;
- struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- int expr_id=0, ret=0;
-
- u_para=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, -1);
- ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, u_para);
- if(ret!=0)
- {
- Maat_region_inner_free(u_para);
- u_para=NULL;
- return -1;
- }
-
- expr_id=scanner->exprid_generator++;
- Maat_region_inner_add_expr_id(u_para, expr_id);
-
- Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para);
- return 0;
-}
-
-
-int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger)
-{
- int i=0;
- struct Maat_table_runtime* table_rt=NULL;
- struct op_expr_t* op_expr=NULL;
-
- UNUSED int ret=0;
- struct Maat_region_inner* region=NULL;
- region=(struct Maat_region_inner*)Maat_hierarchy_region_dettach_user_data(maat_scanner->hier, region_id);
- if(region==NULL)
- {
- return -1;
- }
- ret=Maat_hierarchy_remove_region_from_group(maat_scanner->hier, group_id, region_id);
- assert(ret==0);
- assert(group_id==region->group_id);
-
- switch(table->table_type)
- {
- case TABLE_TYPE_IP:
- case TABLE_TYPE_IP_PLUS:
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- case TABLE_TYPE_INTERVAL:
- case TABLE_TYPE_INTERVAL_PLUS:
- for(i=0;iexpr_id_cnt;i++)
- {
- op_expr=create_op_expr(region->expr_id_lb+i, RULESCAN_OP_DEL, NULL, table->table_id);//del expr
- op_expr->rule_type=rule_type;
- MESA_lqueue_join_tail(maat_scanner->region_update_q, &op_expr, sizeof(void*));
- }
- break;
- case TABLE_TYPE_SIMILARITY:
- case TABLE_TYPE_DIGEST:
- assert(region->expr_id_cnt==1);
- table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id);
- Maat_table_runtime_digest_del(table_rt, region->expr_id_lb);
- break;
- default:
- assert(0);
- break;
- }
- Maat_garbage_bagging(maat_scanner->ref_garbage_bin, region, (void (*)(void*))Maat_region_inner_free);
-
- return 0;
-}
-void update_group2compile_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger)
-{
- struct db_group2compile_rule db_g2c_rule;
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- int ret=0;
- char virtual_table_name[MAX_TABLE_NAME_LEN]={0};
- memset(&db_g2c_rule, 0, sizeof(db_g2c_rule));
- ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%s\t%d", &(db_g2c_rule.group_id),
- &(db_g2c_rule.compile_id),
- &(db_g2c_rule.is_valid),
- &(db_g2c_rule.not_flag),
- virtual_table_name,
- &(db_g2c_rule.clause_index));
- if(ret!=6)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "update error, invalid format of group2compile table %s:%s",
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- return;
- }
- if(db_g2c_rule.clause_index>=MAX_ITEMS_PER_BOOL_EXPR)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "update error, invalid clause index of group2compile table %s:%s",
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- return;
- }
- if(is_valid_table_name(virtual_table_name))
- {
- db_g2c_rule.virtual_table_id=Maat_table_manager_get_id_by_name(table_mgr, virtual_table_name);
- if(db_g2c_rule.virtual_table_id<0)
- {
- //This happens when one data source (e.g. redis) is consumed by multiple Maat instance.
- //Maat ignores unrealated groups.
- MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module,
- "group2compile table load abandon, unknown virtual table name: %s of group table %s:%s.",
- virtual_table_name,
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- return;
- }
- }
- if(db_g2c_rule.is_valid==FALSE)
- {
- ret=Maat_hierarchy_remove_group_from_compile(scanner->hier, db_g2c_rule.group_id,db_g2c_rule.virtual_table_id, db_g2c_rule.not_flag, db_g2c_rule.clause_index, db_g2c_rule.compile_id);
- if(ret==0)
- {
- table_rt->origin_rule_num--;
- assert(table_rt->origin_rule_num>=0);
- if(db_g2c_rule.not_flag)
- {
- table_rt->group2compile.not_flag_group--;
- }
- }
- }
- else
- {
- ret=Maat_hierarchy_add_group_to_compile(scanner->hier, db_g2c_rule.group_id,db_g2c_rule.virtual_table_id, db_g2c_rule.not_flag, db_g2c_rule.clause_index, db_g2c_rule.compile_id);
- if(ret==0)
- {
- table_rt->origin_rule_num++;
- if(db_g2c_rule.not_flag)
- {
- table_rt->group2compile.not_flag_group++;
- }
- }
- }
- return;
-}
-void update_group2group_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger)
-{
- struct db_group2group_rule db_g2g_rule;
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- int ret=0;
- memset(&db_g2g_rule, 0, sizeof(db_g2g_rule));
- ret=sscanf(table_line,"%d\t%d\t%d", &(db_g2g_rule.group_id),
- &(db_g2g_rule.superior_group_id),
- &(db_g2g_rule.is_valid));
- if(ret!=3)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "update error, invalid format of group2group table %s:%s",
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- return;
- }
- if(db_g2g_rule.is_valid==FALSE)
- {
- ret=Maat_hierarchy_remove_group_from_group(scanner->hier, db_g2g_rule.group_id, db_g2g_rule.superior_group_id);
- if(ret==0)
- {
- table_rt->origin_rule_num--;
- assert(table_rt->origin_rule_num>=0);
- }
- }
- else
- {
- ret=Maat_hierarchy_add_group_to_group(scanner->hier, db_g2g_rule.group_id, db_g2g_rule.superior_group_id);
- if(ret==0)
- {
- table_rt->origin_rule_num++;
- }
- }
-
-}
-void update_expr_rule(struct Maat_table_schema* table,const char* table_line,struct Maat_scanner *scanner,void* logger)
-{
- struct db_expr_rule_t* maat_str_rule=ALLOC(struct db_expr_rule_t, 1);
- int ret=0,db_hexbin=0,rule_type=0;
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- switch(table->table_type)
- {
- case TABLE_TYPE_EXPR:
- ret=sscanf(table_line,"%d\t%d\t%1024s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
- ,&(maat_str_rule->group_id)
- ,maat_str_rule->keywords
- ,(int*)&(maat_str_rule->expr_type)
- ,(int*)&(maat_str_rule->match_method)
- ,&db_hexbin
- ,&(maat_str_rule->is_valid));
- if(ret!=7)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "abandon config: invalid format of expr table %s:%s", table->table_name[table->updating_name], table_line);
- free(maat_str_rule);
- maat_str_rule=NULL;
- table->udpate_err_cnt++;
- return;
- }
- break;
- case TABLE_TYPE_EXPR_PLUS:
- ret=sscanf(table_line,"%d\t%d\t%64s\t%1024s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id)
- ,&(maat_str_rule->group_id)
- ,maat_str_rule->district
- ,maat_str_rule->keywords
- ,(int*)&(maat_str_rule->expr_type)
- ,(int*)&(maat_str_rule->match_method)
- ,&db_hexbin
- ,&(maat_str_rule->is_valid));
- if(ret!=8)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "abandon config: invalid format of expr_plus table %s:%s", table->table_name[table->updating_name], table_line);
- free(maat_str_rule);
- maat_str_rule=NULL;
- table->udpate_err_cnt++;
- return;
- }
- break;
- default:
- assert(0);
- break;
- }
- switch(db_hexbin)
- {
- case 0:
- maat_str_rule->is_hexbin=FALSE;
- maat_str_rule->is_case_sensitive=FALSE;
- break;
- case 1:
- maat_str_rule->is_hexbin=TRUE;
- maat_str_rule->is_case_sensitive=FALSE;
- break;
- case 2:
- maat_str_rule->is_hexbin=FALSE;
- maat_str_rule->is_case_sensitive=TRUE;
- break;
- default:
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "abandon config %d:update error,invalid hexbin value of expr table %s:%s",
- maat_str_rule->region_id,
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- if(!is_valid_match_method(maat_str_rule->match_method))
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "abandon config %d:update error,invalid match method=%d in expr table %s:%s",
- maat_str_rule->region_id,
- maat_str_rule->match_method,
- table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- if(!is_valid_expr_type(maat_str_rule->expr_type))
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "abandon config %d:update error,invalid expr type=%d in expr table %s:%s",
- maat_str_rule->region_id,
- maat_str_rule->expr_type,
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
-
- if(maat_str_rule->is_valid==FALSE)
- {
- if(maat_str_rule->expr_type==EXPR_TYPE_REGEX)
- {
- rule_type=RULETYPE_REG;
- }
- else
- {
- rule_type=RULETYPE_STR;
- }
- ret=del_region_rule(table, maat_str_rule->region_id, maat_str_rule->group_id, rule_type,
- scanner, logger);
- if(ret<0)
- {
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num--;
- }
- }
- else
- {
- if(maat_str_rule->expr_type==EXPR_TYPE_AND
- &&maat_str_rule->match_method!=MATCH_METHOD_SUB)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.",
- table->table_name[table->updating_name], maat_str_rule->region_id);
- maat_str_rule->match_method=MATCH_METHOD_SUB;
-
- }
- if(strlen(maat_str_rule->keywords)<4)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module ,
- "Table %s region cfg %d has a expr less than 4 bytes.",
- table->table_name,
- maat_str_rule->region_id);
- }
- ret=add_expr_rule(table, maat_str_rule, scanner, logger);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
- "duplicate config of expr table %s region_id=%d",
- table->table_name[table->updating_name], maat_str_rule->region_id);
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num++;
- }
-
- }
-error_out:
- free(maat_str_rule);
- maat_str_rule=NULL;
-}
-
-void update_ip_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
-{
- struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
- char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0};
- char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0};
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0;
- int protocol=0,direction=0;
- int ret=0;
- int ret_array[8]={1},i=0;
-
- switch(table->table_type)
- {
- case TABLE_TYPE_IP:
- strncpy(saddr_format, "mask", sizeof(saddr_format));
- strncpy(sport_format, "mask", sizeof(sport_format));
- strncpy(daddr_format, "mask", sizeof(daddr_format));
- strncpy(dport_format, "mask", sizeof(dport_format));
-
- ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d",
- &(ip_rule->region_id),
- &(ip_rule->group_id),
- &(ip_rule->addr_type),
- src_ip1,
- src_ip2,
- &src_port1,
- &src_port2,
- dst_ip1,
- dst_ip2,
- &dst_port1,
- &dst_port2,
- &protocol,
- &direction,
- &(ip_rule->is_valid));
- if(ret!=14)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error, invalid column number of ip table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- break;
- case TABLE_TYPE_IP_PLUS:
- ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%s\t%s\t%hu\t%hu\t%s\t%s\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d",
- &(ip_rule->region_id),
- &(ip_rule->group_id),
- &(ip_rule->addr_type),
- saddr_format,
- src_ip1,
- src_ip2,
- sport_format,
- &src_port1,
- &src_port2,
- daddr_format,
- dst_ip1,
- dst_ip2,
- dport_format,
- &dst_port1,
- &dst_port2,
- &protocol,
- &direction,
- &(ip_rule->is_valid));
- if(ret!=18)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error, invalid column number of ip_plus table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- break;
- default:
- table->udpate_err_cnt++;
- goto error_out;
- break;
- }
- if(ip_rule->addr_type!=4&&ip_rule->addr_type!=6)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
- "update error, invalid addr type %d of ip/ip_plus table %s:%s",
- ip_rule->addr_type,
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- if(protocol>65535 || protocol<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
- "update error, invalid protocol value %d of ip/ip_plus table %s:%s",
- protocol,
- table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- if(direction!=0 && direction!=1)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
- "update error, invalid direction value %d of ip/ip_plus table %s:%s",
- direction,
- table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- if(FORMAT_UNKNOWN==ip_format_str2int(saddr_format)||
- FORMAT_UNKNOWN==ip_format_str2int(sport_format)||
- FORMAT_UNKNOWN==ip_format_str2int(daddr_format)||
- FORMAT_UNKNOWN==ip_format_str2int(dport_format))
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
- "update error, invalid addr format of ip/ip_plus table %s:%s, should be range, mask or CIDR",
- table->table_name[table->updating_name], table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
-
- if(ip_rule->addr_type==4)
- {
- ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, &ip_rule->ipv4_rule.min_saddr, &ip_rule->ipv4_rule.max_saddr);
- ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, &ip_rule->ipv4_rule.min_daddr, &ip_rule->ipv4_rule.max_daddr);
-
- if(FORMAT_MASK==ip_format_str2int(sport_format))
- {
- ip_rule->ipv4_rule.min_sport=src_port1&src_port2;
- ip_rule->ipv4_rule.max_sport=src_port1|~src_port2;
- }
- else
- {
- ip_rule->ipv4_rule.min_sport=src_port1;
- ip_rule->ipv4_rule.max_sport=src_port2;
- }
-
- if(FORMAT_MASK==ip_format_str2int(dport_format))
- {
- ip_rule->ipv4_rule.min_dport=dst_port1&dst_port2;
- ip_rule->ipv4_rule.max_dport=dst_port1|~dst_port2;
- }
- else
- {
- ip_rule->ipv4_rule.min_dport=dst_port1;
- ip_rule->ipv4_rule.max_dport=dst_port2;
- }
-
- ip_rule->ipv4_rule.proto=protocol;
- ip_rule->ipv4_rule.direction=direction;
- }
- else
- {
- ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, ip_rule->ipv6_rule.min_saddr, ip_rule->ipv6_rule.max_saddr);
- ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, ip_rule->ipv6_rule.min_daddr, ip_rule->ipv6_rule.max_daddr);
-
- if(FORMAT_MASK==ip_format_str2int(sport_format))
- {
- ip_rule->ipv6_rule.min_sport=src_port1&src_port2;
- ip_rule->ipv6_rule.max_sport=src_port1|~src_port2;
- }
- else
- {
- ip_rule->ipv6_rule.min_sport=src_port1;
- ip_rule->ipv6_rule.max_sport=src_port2;
- }
-
- if(FORMAT_MASK==ip_format_str2int(dport_format))
- {
- ip_rule->ipv6_rule.min_dport=dst_port1&dst_port2;
- ip_rule->ipv6_rule.max_dport=dst_port1|~dst_port2;
- }
- else
- {
- ip_rule->ipv6_rule.min_dport=dst_port1;
- ip_rule->ipv6_rule.max_dport=dst_port2;
- }
-
- ip_rule->ipv6_rule.proto=protocol;
- ip_rule->ipv6_rule.direction=direction;
- }
- for(i=0;i<4;i++)
- {
- if(ret_array[i]<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error, invalid IP address format of ip table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- }
- if(ip_rule->is_valid==FALSE)
- {
- ret=del_region_rule(table,
- ip_rule->region_id, ip_rule->group_id, ip_rule->addr_type==6?RULETYPE_IPv6:RULETYPE_IPv4,
- scanner, logger);
- if(ret<0)
- {
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num--;
- }
- }
- else
- {
-
- ret=add_ip_rule(table, ip_rule, scanner, logger);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "duplicate config of ip table %s config_id=%d"
- ,table->table_name[table->updating_name],ip_rule->region_id);
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num++;
- }
-
- }
-error_out:
- free(ip_rule);
- ip_rule=NULL;
-}
-
-void update_intval_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
-{
- struct db_interval_rule* intval_rule=ALLOC(struct db_interval_rule, 1);
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- int ret=0;
- switch(table->table_type)
- {
- case TABLE_TYPE_INTERVAL:
- ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
- ,&(intval_rule->group_id)
- ,&(intval_rule->intval.lb)
- ,&(intval_rule->intval.ub)
- ,&(intval_rule->is_valid));
-
- if(ret!=5||intval_rule->intval.ubintval.lb)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error,invalid format of interval table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- break;
- case TABLE_TYPE_INTERVAL_PLUS:
- ret=sscanf(table_line,"%d\t%d\t%s\t%u\t%u\t%d",&(intval_rule->region_id)
- ,&(intval_rule->group_id)
- ,intval_rule->district
- ,&(intval_rule->intval.lb)
- ,&(intval_rule->intval.ub)
- ,&(intval_rule->is_valid));
-
- if(ret!=6||intval_rule->intval.ubintval.lb)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error,invalid format of interval table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
- break;
- default:
- assert(0);
- break;
- }
-
-
- if(intval_rule->is_valid==FALSE)
- {
- ret=del_region_rule(table
- ,intval_rule->region_id,intval_rule->group_id,RULETYPE_INT
- ,scanner, logger);
- if(ret<0)
- {
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num--;
- }
-
- }
- else
- {
- ret=add_interval_rule(table, intval_rule,scanner,logger);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "duplicate config of intval table %s config_id=%d"
- ,table->table_name[table->updating_name],intval_rule->region_id);
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num++;
- }
-
- }
-error_out:
- free(intval_rule);
- intval_rule=NULL;
-}
-
-void update_compile_rule(struct Maat_table_schema* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger)
-{
- struct compile_table_schema* compile_desc=&(table->compile);
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
-
- struct Maat_compile_rule *p_compile=NULL;
- struct Maat_rule_head m_rule_tmp;
- memset(&m_rule_tmp, 0, sizeof(m_rule_tmp));
-
- char service_define[MAX_TABLE_LINE_SIZE]={0};
- char tag_str[MAX_TABLE_LINE_SIZE]={0};
- int ret=0;
- int is_valid=0, declared_grp_num=0;
- double exec_seq=0.0;
- ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d\t%lf",&(m_rule_tmp.config_id),
- &(m_rule_tmp.service_id),
- &(m_rule_tmp.action),
- &(m_rule_tmp.do_blacklist),
- &(m_rule_tmp.do_log),
- tag_str,
- service_define,
- &is_valid,
- &declared_grp_num,
- &exec_seq);
- if((ret!=8&&ret!=9&&ret!=10)||declared_grp_num>MAAT_MAX_EXPR_ITEM_NUM)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module ,
- "update error, invalid format of compile table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- return;
-
- }
- if(n_tags>0&&strlen(tag_str)>2)
- {
- str_unescape(tag_str);
- ret=compare_accept_tag(tag_str, tags, n_tags);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "update error, invalid tag format of compile table %s:%s",
- table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- return;
-
- }
- if(ret==0)
- {
- table->unmatch_tag_cnt++;
- return;
- }
- }
- switch(compile_desc->user_region_encoding)
- {
- case USER_REGION_ENCODE_ESCAPE:
- str_unescape(service_define);
- break;
- default:
- break;
- }
-
- if(is_valid==FALSE)
- {
- p_compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_dettach_user_data(scanner->hier, m_rule_tmp.config_id);
- if(p_compile)
- {
- ret=Maat_hierarchy_compile_remove(scanner->hier, m_rule_tmp.config_id);
- assert(ret==0);
- table_rt->origin_rule_num--;
- Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule);
- }
- else
- {
- table->udpate_err_cnt++;
- }
- }
- else
- {
- p_compile=create_compile_rule(&m_rule_tmp, service_define, declared_grp_num, exec_seq, table);
- ret=Maat_hierarchy_compile_add(scanner->hier, m_rule_tmp.config_id, declared_grp_num, p_compile);
- if(ret==0)
- {
- table_rt->origin_rule_num++;
- }
- else
- {
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
- "duplicate config of compile table %s compile_id %d",
- table->table_name[table->updating_name], m_rule_tmp.config_id);
- table->udpate_err_cnt++;
- destroy_compile_rule(p_compile);
- p_compile=NULL;
- table->udpate_err_cnt++;
- }
- }
-
- return;
-}
-
-void update_digest_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger)
-{
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
- struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1);
- int ret=0;
- char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'};
- if(table->table_type==TABLE_TYPE_DIGEST)
- {
- ret=sscanf(table_line,"%d\t%d\t%llu\t%s\t%hd\t%d",&(digest_rule->region_id)
- ,&(digest_rule->group_id)
- ,&(digest_rule->orgin_len)
- ,digest_buff
- ,&(digest_rule->confidence_degree)
- ,&(digest_rule->is_valid));
- }
- else if(table->table_type==TABLE_TYPE_SIMILARITY)
- {
- digest_rule->orgin_len=0;
- ret=sscanf(table_line,"%d\t%d\t%s\t%hd\t%d",&(digest_rule->region_id)
- ,&(digest_rule->group_id)
- ,digest_buff
- ,&(digest_rule->confidence_degree)
- ,&(digest_rule->is_valid));
- }
- else
- {
- assert(0);
- }
- digest_rule->digest_string=digest_buff;
- if(!(ret==6||ret==5)||digest_rule->confidence_degree>100||digest_rule->confidence_degree<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error,invalid format of digest table %s:%s"
- ,table->table_name[table->updating_name],table_line);
- table->udpate_err_cnt++;
- goto error_out;
- }
-
- if(digest_rule->is_valid==FALSE)
- {
- //digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton.
- ret=del_region_rule(table,digest_rule->region_id,digest_rule->group_id,0 ,scanner, logger);
- if(ret<0)
- {
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num--;
- }
-
- }
- else
- {
- ret=add_digest_rule(table, digest_rule,scanner,logger);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "duplicate config of intval table %s config_id=%d"
- ,table->table_name[table->updating_name],digest_rule->region_id);
- table->udpate_err_cnt++;
- }
- else
- {
- table_rt->origin_rule_num++;
- }
-
- }
-error_out:
- digest_rule->digest_string=NULL;
- free(digest_rule);
- digest_rule=NULL;
- return;
-}
-void update_generic_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
-{
- int ret=1, matched_tag=1;
- struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
- char* copy=NULL;
- size_t accept_tag_offset=0, accept_tag_len=0;
- int rule_tag_column=-1;
-
- switch(table_schema->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- rule_tag_column=table_schema->plugin.rule_tag_column;
- break;
- case TABLE_TYPE_IP_PLUGIN:
- rule_tag_column=table_schema->ip_plugin.rule_tag_column;
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- rule_tag_column=table_schema->fqdn_plugin.rule_tag_column;
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- rule_tag_column=table_schema->bool_plugin.rule_tag_column;
- break;
- default:
- assert(0);
- break;
- }
- if(rule_tag_column>0&&n_tags>0)
- {
- ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error, could not locate tag in column %d of table %s:%s",
- rule_tag_column,
- table_schema->table_name[table_schema->updating_name],
- table_row);
- table_schema->udpate_err_cnt++;
- return;
- }
- if(accept_tag_len>2)
- {
- copy=ALLOC(char, accept_tag_len+1);
- memcpy(copy, table_row+accept_tag_offset, accept_tag_len);
- matched_tag=compare_accept_tag(copy, tags, n_tags);
- if(matched_tag<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "update error, invalid tag format of ip_plugin table_schema %s:%s",
- table_schema->table_name[table_schema->updating_name], table_row);
- table_schema->udpate_err_cnt++;
- }
- if(matched_tag==0)
- {
- table_schema->unmatch_tag_cnt++;
- }
- free(copy);
- copy=NULL;
- }
- if(!matched_tag)
- {
- return;
- }
- }
- switch(table_schema->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- Maat_table_runtime_plugin_new_row(table_rt, table_schema, table_row, logger);
- break;
- case TABLE_TYPE_IP_PLUGIN:
- Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, table_row, logger);
- break;
- default:
- assert(0);
- break;
- }
- scanner->xx_plugin_rule_to_update_cnt++;
- return;
-}
-
-void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* logger)
-{
- MESA_htable_handle tmp_map=NULL;
- struct Maat_table_runtime* table_rt=NULL;
- int i=0, ret=0;
- enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID;
-
- ret=Maat_hierarchy_rebuild(scanner->hier);
- if(ret!=0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module ,
- "Version %d hierarchy rebuild failed.",
- scanner->version);
- }
- else
- {
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
- "Version %d hierarchy rebuild success, dedup string rule %lu.",
- scanner->version,
- scanner->dedup_expr_num);
- }
-
- scanner->dedup_expr_num=0;
- rulescan_batch_update(scanner->region,
- scanner->region_update_q,
- logger,
- scanner);
-
- for(i=0; (size_t)imax_table_num; i++)
- {
- table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i);
- if(table_rt==NULL)
- {
- continue;
- }
- table_type=Maat_table_runtime_get_type(table_rt);
- switch(table_type)
- {
- case TABLE_TYPE_DIGEST:
- case TABLE_TYPE_SIMILARITY:
- ret=Maat_table_runtime_digest_batch_udpate(table_rt);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "GIE_update error.");
- }
- break;
- case TABLE_TYPE_PLUGIN:
- Maat_table_runtime_plugin_commit_update(table_rt);
- break;
- case TABLE_TYPE_IP_PLUGIN:
- ret=Maat_table_runtime_ip_plugin_commit_update(table_rt);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "IP plugin table_id %d build failed.", i);
-
- }
-
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- ret=Maat_table_runtime_fqdn_plugin_commit_update(table_rt);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "FQDN plugin table_id %d build failed.", i);
-
- }
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- ret=Maat_table_runtime_bool_plugin_commit_update(table_rt);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Boolean expression plugin table_id %d build failed.", i);
-
- }
- break;
- default:
- break;
- }
-
- }
- if(scanner->tmp_district_map!=NULL)
- {
- tmp_map=scanner->district_map;
- scanner->district_map=scanner->tmp_district_map;
- scanner->tmp_district_map=NULL;
- Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))maat_kv_store_free);
- }
- scanner->last_update_time=time(NULL);
- scanner->xx_plugin_rule_to_update_cnt=0;
- return;
-
-}
-
-void maat_start_cb(long long new_version, int update_type, void*u_para)
-{
- struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
- feather->new_version=new_version;
-
- if(update_type==CM_UPDATE_TYPE_FULL)
- {
- feather->update_tmp_scanner=create_maat_scanner(new_version,feather);
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Full config version %u -> %u update start",
- feather->maat_version,new_version);
- }
- else
- {
-
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Inc config version %u -> %u update start",
- feather->maat_version,new_version);
- feather->maat_version=new_version;
- }
- Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type);
- return;
-}
-long long scanner_rule_num(struct Maat_scanner *scanner)
-{
- long long total=0;
- struct Maat_table_runtime* table_rt=NULL;
- int i=0;
- for(i=0; (size_t)imax_table_num; i++)
- {
- table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i);
- if(table_rt!=NULL)
- {
- total+=table_rt->origin_rule_num;
- }
- }
- return total;
-}
-void maat_finish_cb(void* u_para)
-{
- struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
- long rulescan_wait_q_cnt=0;
-
- Maat_table_manager_all_plugin_cb_finish(feather->table_mgr);
-
- if(feather->update_tmp_scanner!=NULL)
- {
- feather->update_tmp_scanner->cfg_num=scanner_rule_num(feather->update_tmp_scanner);
- do_scanner_update(feather->update_tmp_scanner,
- feather->scan_thread_num,
- feather->logger);
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Full config version %u load %d entries complete.",
- feather->update_tmp_scanner->version,feather->update_tmp_scanner->cfg_num);
- }
- else if(feather->scanner!=NULL)
- {
- feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
- feather->scanner->version=feather->maat_version;
- rulescan_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
- feather->postpone_q_size=rulescan_wait_q_cnt;
- if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
- {
- do_scanner_update(feather->scanner,
- feather->scan_thread_num,
- feather->logger);
- MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module
- ,"Inc config version %u build complete, %d entries in total."
- ,feather->scanner->version,feather->scanner->cfg_num);
- feather->postpone_q_size=0;
- }
- else
- {
- MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module,
- "Postpone %d entries of version %u load to rulescan.",
- feather->scanner->cfg_num, feather->scanner->version);
- }
- }
- else
- {
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Version %d have no valid scan rules, plugin callback complete.",
- feather->maat_version);
- }
- feather->new_version=-1;
- return;
-}
-int maat_update_cb(const char* table_name,const char* line,void *u_para)
-{
- struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
- Maat_scanner* scanner=NULL;
- struct Maat_table_schema* p_table=NULL;
- if(feather->update_tmp_scanner!=NULL)
- {
- scanner=feather->update_tmp_scanner;
- }
- else
- {
- scanner=feather->scanner;
- }
-// MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line);
- p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, table_name);
- if(!p_table)
- {
- MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name);
- return -1;
- }
- Maat_table_schema_set_updating_name(p_table, table_name);
-
- switch(p_table->table_type)
- {
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- update_expr_rule(p_table, line, scanner, feather->logger);
- break;
- case TABLE_TYPE_IP:
- case TABLE_TYPE_IP_PLUS:
- update_ip_rule(p_table, line, scanner, feather->logger);
- break;
- case TABLE_TYPE_INTERVAL:
- case TABLE_TYPE_INTERVAL_PLUS:
- update_intval_rule(p_table, line, scanner,feather->logger);
- break;
- case TABLE_TYPE_DIGEST:
- case TABLE_TYPE_SIMILARITY:
- update_digest_rule(p_table, line, scanner,feather->logger);
- break;
- case TABLE_TYPE_COMPILE:
- update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
- break;
- case TABLE_TYPE_GROUP2COMPILE:
- update_group2compile_rule(p_table, line, scanner, feather->table_mgr, feather->logger);
- break;
- case TABLE_TYPE_GROUP2GROUP:
- update_group2group_rule(p_table, line, scanner, feather->table_mgr, feather->logger);
- break;
- case TABLE_TYPE_PLUGIN:
- case TABLE_TYPE_IP_PLUGIN:
- case TABLE_TYPE_FQDN_PLUGIN:
- case TABLE_TYPE_BOOL_PLUGIN:
- update_generic_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
- break;
- default:
- break;
-
- }
- return 0;
-}
-void *thread_rule_monitor(void *arg)
-{
- struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg;
- struct Maat_scanner* old_scanner=NULL;
- long rulescan_update_wait_q_cnt=0;
- int scan_dir_cnt=0;
- int ret=0;
- char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0};
- char err_str[MAX_TABLE_NAME_LEN]={0};
-
- struct stat attrib;
- size_t total_wait_rule_cnt=0;
-
- char maat_name[16];//Defined by prctl: The name can be up to 16 bytes long,and should
- // be null terminated if it contains fewer bytes.
- if(strlen(feather->instance_name)>0)
- {
- snprintf(maat_name,sizeof(maat_name),"MAAT_%s",feather->instance_name);
- }
- else
- {
- snprintf(maat_name,sizeof(maat_name),"MAAT");
- }
- ret=prctl(PR_SET_NAME,(unsigned long long)maat_name,NULL,NULL,NULL);
- //pthread_setname_np are introduced in glibc2.12
- //ret=pthread_setname_np(pthread_self(),maat_name);
- assert(ret>=0);
-
- pthread_mutex_lock(&(feather->background_update_mutex));
- if(feather->DEFERRED_LOAD_ON!=0)
- {
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Deferred Loading ON, updating in %s.",__func__);
- maat_read_full_config(feather);
- }
- pthread_mutex_unlock(&(feather->background_update_mutex));
- while(feather->is_running)
- {
- usleep(feather->rule_update_checking_interval_ms*1000);
- scan_dir_cnt++;
- if(0==pthread_mutex_trylock(&(feather->background_update_mutex)))
- {
- switch(feather->input_mode)
- {
- case SOURCE_REDIS:
- redis_monitor_traverse(feather->maat_version,
- &(feather->mr_ctx),
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- feather,
- feather->decrypt_key, //Not used.
- feather);
- break;
- case SOURCE_IRIS_FILE:
- config_monitor_traverse(feather->maat_version,
- feather->iris_ctx.inc_dir,
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- feather,
- feather->decrypt_key,
- feather->logger);
- break;
- case SOURCE_JSON_FILE:
- memset(md5_tmp, 0, sizeof(md5_tmp));
- stat(feather->json_ctx.json_file, &attrib);
- if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim)))
- {
- feather->json_ctx.last_md5_time=attrib.st_ctim;
- md5_file(feather->json_ctx.json_file, md5_tmp);
- if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5))
- {
- ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str));
- if(ret<0)
- {
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,
- "Maat re-initiate with JSON file %s (md5=%s)failed: %s",
- feather->json_ctx.json_file,
- md5_tmp,
- err_str);
- }
- else
- {
- config_monitor_traverse(0,
- feather->json_ctx.iris_file,
- maat_start_cb,
- maat_update_cb,
- maat_finish_cb,
- feather,
- feather->decrypt_key,
- feather->logger);
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,
- "Maat re-initiate with JSON file %s success, md5: %s",
- feather->json_ctx.json_file,
- md5_tmp);
-
- }
-
- }
- }
- break;
- default:
- assert(0);
- break;
- }
-
- if(feather->update_tmp_scanner!=NULL)
- {
- old_scanner=feather->scanner;
- //Some OS doesn't have __sync_lock_test_and_set.
- //feather->scanner=__sync_lock_test_and_set(&(feather->scanner),feather->update_tmp_scanner);
- feather->scanner=feather->update_tmp_scanner;
- if(old_scanner!=NULL)
- {
- if(feather->scanner->version>old_scanner->version)
- {
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Maat version updated %d -> %d.",
- old_scanner->version, feather->scanner->version);
- }
- else
- {
- MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module,
- "Maat version roll back %d -> %d.",
- old_scanner->version, feather->scanner->version);
- }
- feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num);
- Maat_garbage_bagging(feather->garbage_bin, old_scanner, (void (*)(void*))destroy_maat_scanner);
- }
- feather->update_tmp_scanner=NULL;
- feather->maat_version=feather->scanner->version;
- feather->last_full_version=feather->scanner->version;
- }
- if(feather->scanner!=NULL)
- {
- rulescan_update_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
- feather->postpone_q_size=rulescan_update_wait_q_cnt;
- total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->xx_plugin_rule_to_update_cnt;
- if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
- {
- do_scanner_update(feather->scanner,
- feather->scan_thread_num,
- feather->logger);
- feather->postpone_q_size=0;
- MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
- "Actual udpate config version %u, %d entries load to rulescan after postpone.",
- feather->scanner->version,feather->scanner->cfg_num);
- }
- }
- pthread_mutex_unlock(&(feather->background_update_mutex));
- }
- Maat_garbage_collect_routine(feather->garbage_bin);
- if(feather->stat_on==1&&time(NULL)%2==0)//output every 2 seconds
- {
- maat_stat_output(feather);
- }
- }
-
- destroy_maat_scanner(feather->scanner);
- Maat_garbage_bin_free(feather->garbage_bin);
- Maat_table_manager_destroy(feather->table_mgr);//Table manager MUST be freed at last.
-
- alignment_int64_array_free(feather->thread_call_cnt);
- alignment_int64_array_free(feather->compile_mid_cnt);
- alignment_int64_array_free(feather->outer_mid_cnt);
- alignment_int64_array_free(feather->hit_cnt);
- alignment_int64_array_free(feather->not_grp_hit_cnt);
- if(feather->input_mode==SOURCE_REDIS)
- {
- if(feather->mr_ctx.read_ctx)
- {
- redisFree(feather->mr_ctx.read_ctx);
- feather->mr_ctx.read_ctx=NULL;
- }
- if(feather->mr_ctx.write_ctx)
- {
- redisFree(feather->mr_ctx.write_ctx);
- feather->mr_ctx.write_ctx=NULL;
- }
- }
- int i=0;
- for(i=0; in_tags; i++)
- {
- free(feather->accept_tags[i].tag_name);
- free(feather->accept_tags[i].tag_val);
- }
- free(feather->accept_tags);
- free(feather->accept_tags_raw);
- if(feather->stat_on&& feather->stat_handle)
- {
- FS_stop(&(feather->stat_handle));
- }
- free(feather);
- return NULL;
-}
-
diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp
deleted file mode 100644
index 78e0307..0000000
--- a/src/entry/Maat_stat.cpp
+++ /dev/null
@@ -1,368 +0,0 @@
-#include "Maat_rule_internal.h"
-#include "Maat_table_schema.h"
-#include "Maat_garbage_collection.h"
-#include "alignment_int64.h"
-#include
-#include
-enum MAAT_FS_STATUS{
- STATUS_VERSION=0,
- STATUS_THRED_NUM,
- STATUS_TABLE_NUM,
- STATUS_PLUGIN_CACHE_NUM,
- STATUS_PLUGIN_ACC_NUM,
- STATUS_GROUP_REF_NUM,
- STATUS_GROUP_REF_NOT_NUM,
- STATUS_COMPILE_RULE_NUM,
- STATUS_POSTPONE_QSIZE,
- STATUS_OUTER_MID_NUM,
- STATUS_INNER_MID_NUM,
- STATUS_GARBAGE_QSIZE,
- STATUS_TOTAL_SCAN_LEN,
- STATUS_TOTAL_SCAN_CNT,
- STATUS_UPDATE_ERR_CNT,
- STATUS_ICONV_ERR_CNT,
- STATUS_SCAN_ERR_CNT,
- STATUS_ZOMBIE_RS_STREAM,
- STATUS_NOT_GROUP_HIT,
- STATUS_CMD_NUM,
- STATUS_CMD_Q_SIZE,
- STATUS_CMD_LINE_NUM
-};
-
-enum MAAT_FS_COLUMN
-{
- COLUMN_TABLE_RULE_NUM=0,
- COLUMN_TABLE_REGEX_NUM,
- COLUMN_TABLE_STREAM_NUM,
- COLUMN_TABLE_SCAN_CNT,
- COLUMN_TABLE_SCAN_BYTES,
- COLUMN_TABLE_CPU_TIME,//microseconds
- COLUMN_TABLE_HIT_CNT,
-};
-#define MAX_CONJ_NAME_LEN 22
-void maat_stat_init(struct _Maat_feather_t* feather)
-{
- int value=0;
- int i=0,j=0,offset=0;
- struct Maat_table_schema* p_table=NULL;
- char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0};
-
- feather->stat_handle=FS_create_handle();
- FS_set_para(feather->stat_handle, OUTPUT_DEVICE, feather->stat_file, strlen(feather->stat_file)+1);
- value=1;
- FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value));
- value=0;
- FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value));
-
- FS_set_para(feather->stat_handle, APP_NAME, feather->instance_name, strlen(feather->instance_name)+1);
- FS_set_para(feather->stat_handle, OUTPUT_PROMETHEUS, &feather->output_prometheus, sizeof(feather->output_prometheus));
-
- feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "version");
- feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "threads");
- feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "tables");
-
- feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "plug_cached");
- feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "plug_acc");
-
- feather->fs_status_id[STATUS_GROUP_REF_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "group");
- feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "not_grp");
- feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "compile");
-
- feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "postponed");
- feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "garbage_num");
-
- feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "outer_mid");
- feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "inner_mid");
- feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "z_stream");
-
- feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "nt_grp_hit");
-
- feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_bytes");
- feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_times");
- feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "update_err");
- feather->fs_status_id[STATUS_ICONV_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "iconv_err");
- feather->fs_status_id[STATUS_SCAN_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_error");
- feather->fs_status_id[STATUS_CMD_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "cmd_commit");
- feather->fs_status_id[STATUS_CMD_Q_SIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "cmd_in_q");
- feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED, "line_cmd/s");
-
- feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "rule");
- feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "reg/v6");
- feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "stream");
- feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "IN_Bps");
- if(feather->perf_on==1)
- {
- feather->fs_column_id[COLUMN_TABLE_CPU_TIME]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "cpu_us");
- value=feather->fs_column_id[COLUMN_TABLE_CPU_TIME];
- FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
- FS_register_ratio(feather->stat_handle,
- feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
- feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
- 1000000, //microsecond to second
- FS_STYLE_COLUMN,
- FS_CALC_SPEED,
- "PROC_Bps");
- }
- feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "IN_Tps");
- if(feather->perf_on==1)
- {
- FS_register_ratio(feather->stat_handle,
- feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
- feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
- 1000000, //microsecond to second
- FS_STYLE_COLUMN,
- FS_CALC_SPEED,
- "PROC_Tps");
- }
- feather->fs_column_id[COLUMN_TABLE_HIT_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "hit_cnt");
- value=feather->fs_column_id[COLUMN_TABLE_HIT_CNT];
- FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value));
- FS_register_ratio(feather->stat_handle,
- feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
- feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
- 1,
- FS_STYLE_COLUMN,
- FS_CALC_SPEED,
- "hit_rate");
- feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum");
- size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
- for(i=0; i<(int)max_table_num; i++)
- {
- p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i);
- if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
- ||p_table->table_type==TABLE_TYPE_GROUP
- ||p_table->table_type==TABLE_TYPE_COMPILE)
- {
- continue;
- }
- offset=0;
- for(j=0;jconj_cnt;j++)
- {
- offset+=snprintf(conj_table_name+offset,sizeof(conj_table_name)-offset,
- "%s/", p_table->table_name[j]);
- }
- conj_table_name[offset-1]='\0';//delete the last slash
- if(strlen(conj_table_name)>MAX_CONJ_NAME_LEN)
- {
- conj_table_name[MAX_CONJ_NAME_LEN]='\0';
- }
- p_table->stat_line_id=FS_register(feather->stat_handle,
- FS_STYLE_LINE, FS_CALC_CURRENT,
- conj_table_name);
- }
- FS_start(feather->stat_handle);
- return;
-}
-
-void maat_stat_output(struct _Maat_feather_t* feather)
-{
- long value=0;
- long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
- long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
- long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
- long long outer_mid_cnt=0,inner_mid_cnt=0;
- long long not_grp_hit_cnt=0;
- long long total_update_error=0,total_iconv_error=0;
- long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0;
- int i=0;
- time_t now;
- struct Maat_table_schema* p_table=NULL;
- struct Maat_table_runtime* table_rt=NULL;
- time(&now);
- if(feather->scanner==NULL)
- {
- return;
- }
- active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
- outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt, feather->scan_thread_num);
- inner_mid_cnt=alignment_int64_array_sum(feather->compile_mid_cnt, feather->scan_thread_num);
- not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt, feather->scan_thread_num);
-
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0, FS_OP_SET, feather->maat_version);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0, FS_OP_SET, active_thread_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0, FS_OP_SET, Maat_table_manager_get_count(feather->table_mgr));
-
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0, FS_OP_SET, outer_mid_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0, FS_OP_SET, inner_mid_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0, FS_OP_SET, not_grp_hit_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0, FS_OP_SET, feather->cmd_acc_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0, FS_OP_SET, feather->cmd_q_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_LINE_NUM], 0, FS_OP_SET, feather->line_cmd_acc_num);
-
- value=Maat_garbage_bin_get_size(feather->garbage_bin);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
- feather->update_err_cnt=0;
- feather->iconv_err_cnt=0;
- size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
- for(i=0; i<(int)max_table_num; i++)
- {
- table_stream_num=0;
- table_scan_cnt=0;
- table_input_bytes=0;
- table_scan_cpu_time=0;
- table_hit_cnt=0;
- table_regex_ipv6_num=0;
-
- p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i);
- if(p_table==NULL)
- {
- continue;
- }
- table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i);
- switch(p_table->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- plugin_cache_num+=Maat_table_runtime_plugin_cached_row_count(table_rt);
- plugin_acc_num+=table_rt->plugin.acc_line_num;
- break;
- case TABLE_TYPE_GROUP:
- group_rule_num+=table_rt->origin_rule_num;
- not_group_rule_num+=table_rt->group2compile.not_flag_group;
- break;
- case TABLE_TYPE_COMPILE:
- compile_rule_num+=table_rt->origin_rule_num;
- break;
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- table_regex_ipv6_num=table_rt->expr.regex_rule_cnt;
- total_iconv_error=p_table->expr.iconv_err_cnt;
- break;
- case TABLE_TYPE_IP:
- case TABLE_TYPE_IP_PLUS:
- table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt;
- break;
- default:
- break;
- }
- if(p_table->table_type==TABLE_TYPE_PLUGIN||
- p_table->table_type==TABLE_TYPE_GROUP||
- p_table->table_type==TABLE_TYPE_COMPILE)
- {
- continue;
- }
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
- FS_OP_SET,
- table_rt->origin_rule_num);
- total_cfg_num+=table_rt->origin_rule_num;
-
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
- FS_OP_SET,
- table_regex_ipv6_num);
- total_regex_num+=table_regex_ipv6_num;
-
- table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num);
- alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num);
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
- FS_OP_ADD,
- table_stream_num);
- total_stream_cnt+= table_stream_num;
-
-
- table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num);
- alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num);
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
- FS_OP_ADD,
- table_scan_cnt);
- total_scan_cnt+=table_scan_cnt;
-
- table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num);
- alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num);
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
- FS_OP_ADD,
- table_input_bytes);
- total_input_bytes+=table_input_bytes;
- if(feather->perf_on==1)
- {
- table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num);
- alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num);
- table_scan_cpu_time/=1000;
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
- FS_OP_ADD,
- table_scan_cpu_time);
- total_cpu_time+=table_scan_cpu_time;
- }
-
- table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num);
- alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num);
- FS_operate(feather->stat_handle,
- p_table->stat_line_id,
- feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
- FS_OP_ADD,
- table_hit_cnt);
- total_update_error+=p_table->udpate_err_cnt;
- }
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
- FS_OP_SET,
- total_cfg_num);
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
- FS_OP_SET,
- total_regex_num);
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
- FS_OP_ADD,
- total_stream_cnt);
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
- FS_OP_ADD,
- total_scan_cnt);
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
- FS_OP_ADD,
- total_input_bytes);
- if(feather->perf_on==1)
- {
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
- FS_OP_ADD,
- total_cpu_time);
- }
- total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num);
- alignment_int64_array_reset(feather->hit_cnt,feather->scan_thread_num);
- FS_operate(feather->stat_handle,
- feather->total_stat_id,
- feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
- FS_OP_ADD,
- total_hit_cnt);
- feather->total_scan_bytes+=total_input_bytes;
- feather->total_scan_cnt+=total_scan_cnt;
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_LEN], 0,FS_OP_SET,feather->total_scan_bytes);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_CNT], 0,FS_OP_SET,feather->total_scan_cnt);
- feather->update_err_cnt=total_update_error;
- feather->iconv_err_cnt=total_iconv_error;
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_UPDATE_ERR_CNT], 0,FS_OP_SET,feather->update_err_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ICONV_ERR_CNT], 0,FS_OP_SET,feather->iconv_err_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream);
-
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NUM], 0,FS_OP_SET,group_rule_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM], 0,FS_OP_SET,not_group_rule_num);
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num);
-
- FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size);
-
- FS_passive_output(feather->stat_handle);
- return;
-}
-
diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp
deleted file mode 100644
index feb7bc2..0000000
--- a/src/entry/Maat_table_runtime.cpp
+++ /dev/null
@@ -1,980 +0,0 @@
-#include "Maat_rule.h"
-#include "Maat_utils.h"
-#include "Maat_table_runtime.h"
-#include
-#include
-#include
-
-
-
-#include "dynamic_array.h"
-#include
-#include
-
-struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row)
-{
- struct ip_rule* range_rule=ALLOC(struct ip_rule, 1);
-
- int ret[4]={0};
- size_t column_offset=0, column_len=0;
- char start_ip[128]={0}, end_ip[128]={0};
- ret[0]=get_column_pos(row, schema->row_id_column, &column_offset, &column_len);
- range_rule->rule_id=atoi(row+column_offset);
-
- ret[1]=get_column_pos(row, schema->ip_type_column, &column_offset, &column_len);
- int ip_type=atoi(row+column_offset);
-
- ret[2]=get_column_pos(row, schema->start_ip_column, &column_offset, &column_len);
- strncpy(start_ip, row+column_offset, MIN(column_len, sizeof(start_ip)));
-
- ret[3]=get_column_pos(row, schema->end_ip_column, &column_offset, &column_len);
- strncpy(end_ip, row+column_offset, MIN(column_len, sizeof(end_ip)));
- if(ret[0]<0||ret[1]<0||ret[2]<0||ret[3]<0)
- {
- free(range_rule);
- return NULL;
- }
-
- if(ip_type==4)
- {
- ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, &(range_rule->ipv4_rule.start_ip), &(range_rule->ipv4_rule.end_ip));
- range_rule->type=IPv4;
- }
- else if(ip_type==6)
- {
- ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, range_rule->ipv6_rule.start_ip, range_rule->ipv6_rule.end_ip);
- range_rule->type=IPv6;
- }
- else
- {
- free(range_rule);
- return NULL;
- }
- if(ret[0]<0)
- {
- free(range_rule);
- return NULL;
- }
- range_rule->user_tag=NULL;
- return range_rule;
-}
-void ip_rule_free(struct ip_rule* p)
-{
- free(p);
- return;
-}
-static int cmp_ull_p(const void *p1, const void *p2)
-{
- if(* (unsigned long long*) p1 > * (unsigned long long*) p2)
- {
- return 1;
- }
- else if(* (unsigned long long*) p1 < * (unsigned long long*) p2)
- {
- return -1;
- }
- else
- {
- return 0;
- }
-}
-size_t ull_dedup(unsigned long long item_ids[], size_t n_item)
-{
- qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p);
-
- size_t J=0;
- for(size_t i=1; iid=id;
- rule->operation=op;
- if(digest!=NULL)
- {
- digest_len=strlen(digest);
- rule->sfh=(char*)calloc(sizeof(char),digest_len+1);
- memcpy(rule->sfh,digest,digest_len);
-
- }
- rule->sfh_length=digest_len;
- rule->cfds_lvl=cfds_lvl;
- rule->tag=tag;
- return rule;
-}
-static void destroy_digest_rule(GIE_digest_t*rule)
-{
- if(rule->sfh!=NULL)
- {
- free(rule->sfh);
- rule->sfh=NULL;
- }
- free(rule);
- rule=NULL;
- return;
-}
-struct FQDN_rule *fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
-{
- struct FQDN_rule *fqdn_rule=ALLOC(struct FQDN_rule, 1);
- //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$
- if(fqdn[0]=='.')
- {
- fqdn++;
- fqdn_len--;
- }
- if(fqdn[fqdn_len]=='/')
- {
- fqdn_len--;
- }
- fqdn_rule->FQDN=ALLOC(char, fqdn_len+1);
- memcpy(fqdn_rule->FQDN, fqdn, fqdn_len);
- fqdn_rule->len=fqdn_len;
- fqdn_rule->is_suffix_match=is_suffix_match;
- fqdn_rule->id=id;
- return fqdn_rule;
-}
-void fqdn_rule_free(struct FQDN_rule *fqdn_rule)
-{
- assert(fqdn_rule->user_tag==NULL);
- free(fqdn_rule->FQDN);
- fqdn_rule->FQDN=NULL;
- free(fqdn_rule);
- return;
-}
-struct bool_expr *bool_expr_new(unsigned int id, unsigned long long item_ids[], size_t n_item)
-{
- struct bool_expr *expr=ALLOC(struct bool_expr, 1);
- expr->expr_id=id;
- size_t i=0;
- n_item=ull_dedup(item_ids, n_item);
- for(i=0; iitems[i].item_id=item_ids[i];
- expr->items[i].not_flag=0;
- }
- expr->item_num=n_item;
- return expr;
-}
-void bool_expr_free(struct bool_expr *expr)
-{
- free(expr);
- return;
-}
-void _notype_fqdn_rule_free(void* p)
-{
- fqdn_rule_free((struct FQDN_rule*)p);
- return;
-}
-static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num, struct Maat_garbage_bin* bin)
-{
-
- struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
- table_rt->table_type=table_schema->table_type;
- switch(table_schema->table_type)
- {
- case TABLE_TYPE_DIGEST:
- case TABLE_TYPE_SIMILARITY:
- table_rt->similar.update_q=MESA_lqueue_create(0,0);
- break;
- case TABLE_TYPE_PLUGIN:
- table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->plugin.ex_schema.key2index_func,
- NULL);
- if(table_schema->plugin.ex_schema.set_flag)
- {
- EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
- }
- break;
- case TABLE_TYPE_IP_PLUGIN:
- table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->ip_plugin.ex_schema.key2index_func,
- free);
- if(table_schema->ip_plugin.ex_schema.set_flag)
- {
- EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
- }
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->fqdn_plugin.ex_schema.key2index_func,
- _notype_fqdn_rule_free);
- if(table_schema->fqdn_plugin.ex_schema.set_flag)
- {
- EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
- }
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- table_rt->bool_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->fqdn_plugin.ex_schema.key2index_func,
- (void (*)(void*))bool_expr_free);
- if(table_schema->bool_plugin.ex_schema.set_flag)
- {
- EX_data_rt_set_schema(table_rt->bool_plugin.ex_data_rt, &table_schema->bool_plugin.ex_schema);
- }
- break;
- default:
- break;
- }
- table_rt->ref_garbage_bin=bin;
- table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
- table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
- table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
- table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
- table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
- return table_rt;
-}
-static void table_runtime_free(struct Maat_table_runtime* p)
-{
- long q_cnt=0,data_size=0;
- int i=0;
- UNUSED int q_ret=0;
-
- GIE_digest_t* digest_rule=NULL;
- if(p==NULL)
- {
- return;
- }
- switch(p->table_type)
- {
- case TABLE_TYPE_DIGEST:
- case TABLE_TYPE_SIMILARITY:
- if(p->similar.gie_handle!=NULL)
- {
- GIE_destory(p->similar.gie_handle);
- }
- if(p->similar.update_q!=NULL)
- {
- q_cnt=MESA_lqueue_get_count(p->similar.update_q);
- for(i=0;isimilar.update_q,&digest_rule,&data_size);
- assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
- destroy_digest_rule(digest_rule);
- }
- MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL);
- }
- break;
-
- case TABLE_TYPE_IP_PLUGIN:
- ip_matcher_free(p->ip_plugin.ip_matcher);
- EX_data_rt_free(p->ip_plugin.ex_data_rt);
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- FQDN_engine_free(p->fqdn_plugin.fqdn_engine);
- EX_data_rt_free(p->fqdn_plugin.ex_data_rt);
- break;
- case TABLE_TYPE_PLUGIN:
- EX_data_rt_free(p->plugin.ex_data_rt);
- break;
- default:
- break;
- }
-
- alignment_int64_array_free(p->scan_cnt);
- alignment_int64_array_free(p->scan_cpu_time);
- alignment_int64_array_free(p->input_bytes);
- alignment_int64_array_free(p->stream_num);
- alignment_int64_array_free(p->hit_cnt);
- free(p);
- return;
-}
-
-struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin)
-{
- const struct Maat_table_schema* table_desc=NULL;
- struct Maat_table_runtime* table_rt=NULL;
- struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1);
- size_t i=0;
- table_rt_mgr->ref_bin=bin;
- table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager);
- table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
- for(i=0; in_table_rt; i++)
- {
- table_desc=Maat_table_manager_get_by_id_raw(table_manager, i);
- if(!table_desc)
- {
- continue;
- }
- table_rt=table_runtime_new(table_desc, max_thread_num, table_rt_mgr->ref_bin);
- table_rt_mgr->table_rt[i]=table_rt;
- }
- return table_rt_mgr;
-}
-void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr)
-{
- size_t i=0;
- for(i=0; in_table_rt; i++)
- {
- table_runtime_free(table_rt_mgr->table_rt[i]);
- table_rt_mgr->table_rt[i]=NULL;
- }
- free(table_rt_mgr->table_rt);
- table_rt_mgr->table_rt=NULL;
- table_rt_mgr->ref_bin=NULL;
- free(table_rt_mgr);
-}
-struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id)
-{
- assert(table_id<(int)table_rt_mgr->n_table_rt);
- return table_rt_mgr->table_rt[table_id];
-}
-enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* table_rt)
-{
- return table_rt->table_type;
-}
-void Maat_table_runtime_perf_stat(struct Maat_table_runtime* p, int scan_len, struct timespec* start, struct timespec* end,int thread_num)
-{
- alignment_int64_array_add(p->scan_cnt,thread_num,1);
- alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
- if(start!=NULL&&end!=NULL)
- {
- alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
- }
- return;
-}
-
-size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt)
-{
- struct plugin_runtime* plugin_rt=&(table_rt->plugin);
- assert(table_rt->table_type==TABLE_TYPE_PLUGIN);
- return EX_data_rt_get_cached_row_num(plugin_rt->ex_data_rt);
-}
-const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row)
-{
- const char* line=NULL;
- struct plugin_runtime* plugin_rt=&(table_rt->plugin);
- line=EX_data_rt_cached_row_get(plugin_rt->ex_data_rt, Nth_row);
- return line;
-}
-MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key)
-{
- MAAT_RULE_EX_DATA ex_data=NULL;
- if(!table_schema->plugin.ex_schema.set_flag)
- {
- assert(0);
- return NULL;
- }
- ex_data=EX_data_rt_get_EX_data_by_key(table_rt->plugin.ex_data_rt, key, strlen(key));
- return ex_data;
-
-}
-void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
-{
- int ret=0, i=0;
- size_t is_valid_offset=0, valid_len=0;
- size_t key_offset=0, key_len=0;
-
- struct plugin_table_schema* plugin_schema=&table_schema->plugin;
- struct plugin_runtime* plugin_rt=&table_rt->plugin;
- ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
- plugin_rt->acc_line_num++;
- if(plugin_schema->ex_schema.set_flag)
- {
- ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "plugin EX data process error: cannot find column %d of %s",
- plugin_schema->key_column, row);
- return;
- }
- if(atoi(row+is_valid_offset)==1)
- {
- EX_data_rt_row2EX_data(plugin_rt->ex_data_rt, row, row+key_offset, key_len, NULL, logger);
- }
- else
- {
- EX_data_rt_delete_by_row(plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger);
- }
- }
- if(plugin_schema->cb_plug_cnt>0)
- {
- for(i=0; icb_plug_cnt;i++)
- {
- plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para);
- }
- }
- if(!plugin_schema->ex_schema.set_flag && !plugin_schema->cb_plug_cnt)
- {
- EX_data_rt_cache_row_put(plugin_rt->ex_data_rt, row);
- }
-
- return;
-}
-void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt)
-{
- EX_data_rt_update_commit(table_rt->plugin.ex_data_rt);
- table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt);
- return;
-}
-
-void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
-{
- GIE_digest_t* digest_rule=NULL;
- char *dup_digest=_maat_strdup(digest);
-
- if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
- {
- dup_digest=str_unescape(dup_digest);
- }
- digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT,
- dup_digest,
- confidence_degree,
- tag);
- MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
- free(dup_digest);
- return;
-}
-void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id)
-{
- GIE_digest_t* digest_rule=NULL;
- digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest
- ,NULL
- ,0
- ,NULL);
- MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
- return;
-}
-
-void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
-{
- struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin);
- struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
- size_t is_valid_offset=0, valid_len=0;
- size_t is_suffix_flag_offset=0, is_suffix_flag_len=0;
- size_t row_id_offset=0, row_id_len=0;
- size_t fqdn_offset=0, fqdn_len=0;
- struct FQDN_rule* fqdn_rule=NULL;
- int ret=0;
- if(fqdn_plugin_schema->ex_schema.set_flag)
- {
- ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "fqdn_plugin EX data process error: cannot find is_valid column %d of %s",
- fqdn_plugin_schema->valid_flag_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "fqdn_plugin EX data process error: cannot find row id column %d of %s",
- fqdn_plugin_schema->row_id_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s",
- fqdn_plugin_schema->is_suffix_flag_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "fqdn_plugin EX data process error: cannot find fqdn column %d of %s",
- fqdn_plugin_schema->fqdn_column, row);
- return;
- }
-
- if(atoi(row+is_valid_offset)==1)//add
- {
- fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset));
- ret=EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger);
- if(ret<0)
- {
- fqdn_rule_free(fqdn_rule);
- fqdn_rule=NULL;
- }
- }
- else
- {
- EX_data_rt_delete_by_row(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
- }
- }
- else
- {
- EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row);
- table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt);
- }
- fqdn_plugin_rt->changed_flag=1;
- return;
-}
-
-int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt)
-{
- struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL;
- struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
- assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
- struct EX_data_container **exc_array=NULL;
- struct FQDN_rule* rules=NULL;
- size_t rule_cnt=0, i=0, ret=0;
- if(!fqdn_rt->changed_flag)
- {
- return ret;
- }
- rule_cnt=EX_data_rt_list_updating_ex_containers(fqdn_rt->ex_data_rt, &exc_array);
- rules=ALLOC(struct FQDN_rule, rule_cnt);
- for(i=0; i0)
- {
- new_fqdn_engine=FQDN_engine_new(rules, rule_cnt);
- if(!new_fqdn_engine)
- {
- ret=-1;
- }
- }
- old_fqdn_engine=fqdn_rt->fqdn_engine;
- fqdn_rt->fqdn_engine=new_fqdn_engine;
- Maat_garbage_bagging(table_rt->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
- EX_data_rt_update_commit(fqdn_rt->ex_data_rt);
- table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(fqdn_rt->ex_data_rt);
-
- free(rules);
- free(exc_array);
- table_rt->fqdn_plugin.changed_flag=0;
- return ret;
-}
-struct EX_data_rt *Maat_table_runtime_get_EX_data_rt(struct Maat_table_runtime* table_rt)
-{
- struct EX_data_rt *rt=NULL;
-
- switch(table_rt->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- rt=table_rt->plugin.ex_data_rt;
- break;
- case TABLE_TYPE_IP_PLUGIN:
- rt=table_rt->ip_plugin.ex_data_rt;
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- rt=table_rt->fqdn_plugin.ex_data_rt;
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- rt=table_rt->bool_plugin.ex_data_rt;
- break;
- default:
- break;
- }
- return rt;
-}
-int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
-{
- size_t i=0;
- const char* row=NULL;
- assert(table_rt->table_type==table_schema->table_type);
- struct EX_data_rt *ex_data_rt=NULL;
- struct EX_data_schema *ex_data_schema=Maat_table_schema_get_EX_data_schema(table_schema);
- ex_data_rt=Maat_table_runtime_get_EX_data_rt(table_rt);
- EX_data_rt_set_schema(ex_data_rt, ex_data_schema);
- for(i=0; itable_type)
- {
- case TABLE_TYPE_PLUGIN:
- Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
- break;
- case TABLE_TYPE_IP_PLUGIN:
- Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, row, logger);
- break;
- default:
- break;
- }
- }
- EX_data_rt_clear_row_cache(ex_data_rt);
- switch(table_rt->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- Maat_table_runtime_plugin_commit_update(table_rt);
- break;
- case TABLE_TYPE_IP_PLUGIN:
- Maat_table_runtime_ip_plugin_commit_update(table_rt);
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- Maat_table_runtime_fqdn_plugin_commit_update(table_rt);
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- Maat_table_runtime_bool_plugin_commit_update(table_rt);
- break;
- default:
- break;
- }
- return 0;
-}
-
-int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
-{
- struct FQDN_match results[size];
- int n_result=0, i=0;
- if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
- {
- return -1;
- }
- if(!table_rt->fqdn_plugin.fqdn_engine)
- {
- return 0;
- }
- n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size);
- for(i=0; ifqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
- }
- return n_result;
-}
-
-void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
-{
- struct bool_plugin_table_schema *bool_plugin_schema=&(table_schema->bool_plugin);
- struct bool_plugin_runtime *bool_plugin_rt=&(table_rt->bool_plugin);
- size_t is_valid_offset=0, valid_len=0;
- size_t row_id_offset=0, row_id_len=0;
- size_t bool_expr_offset=0, bool_expr_len=0;
- struct bool_expr *expr=NULL;
- int ret=0;
- unsigned long long item_id[MAX_ITEMS_PER_BOOL_EXPR];
- size_t n_item=0;
- if(bool_plugin_schema->ex_schema.set_flag)
- {
- ret=Maat_helper_read_column(row, bool_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "bool_plugin EX data process error: cannot find is_valid column %d of %s",
- bool_plugin_schema->valid_flag_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, bool_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "bool_plugin EX data process error: cannot find row id column %d of %s",
- bool_plugin_schema->row_id_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, bool_plugin_schema->bool_expr_column, &bool_expr_offset, &bool_expr_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "bool_plugin EX data process error: cannot find bool_expr column %d of %s",
- bool_plugin_schema->bool_expr_column, row);
- return;
- }
- char *token=NULL, *sub_token=NULL, *saveptr;
- char expr_buffer[256];
- memset(expr_buffer, 0, sizeof(expr_buffer));
- memcpy(expr_buffer, row+bool_expr_offset, bool_expr_len);
- for (token = expr_buffer; ; token= NULL)
- {
- sub_token= strtok_r(token, "&", &saveptr);
- if (sub_token == NULL)
- break;
- ret=sscanf(sub_token, "%llu", item_id+n_item);
- n_item++;
- if(ret!=1||n_item>MAX_ITEMS_PER_BOOL_EXPR)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "bool_plugin EX data process error: invalid format of bool_expr column %d of %s",
- bool_plugin_schema->bool_expr_column, row);
- return;
- }
- }
- if(atoi(row+is_valid_offset)==1)//add
- {
- expr=bool_expr_new((unsigned int)atoi(row+row_id_offset), item_id, n_item);
- ret=EX_data_rt_row2EX_data(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, expr, logger);
- if(ret<0)
- {
- bool_expr_free(expr);
- expr=NULL;
- }
- }
- else
- {
- EX_data_rt_delete_by_row(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
- }
- }
- else
- {
- EX_data_rt_cache_row_put(bool_plugin_rt->ex_data_rt, row);
- table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(bool_plugin_rt->ex_data_rt);
- }
- bool_plugin_rt->changed_flag=1;
- return;
-}
-int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt)
-{
- struct bool_matcher* new_bool_matcher=NULL, *old_bool_matcher=NULL;
- struct bool_plugin_runtime* bool_plugin_rt=&table_rt->bool_plugin;
- assert(table_rt->table_type==TABLE_TYPE_BOOL_PLUGIN);
- struct EX_data_container **exc_array=NULL;
- struct bool_expr* exprs=NULL;
- size_t expr_cnt=0, i=0, ret=0, mem_usage=0;
- if(!bool_plugin_rt->changed_flag)
- {
- return ret;
- }
- expr_cnt=EX_data_rt_list_updating_ex_containers(bool_plugin_rt->ex_data_rt, &exc_array);
- exprs=ALLOC(struct bool_expr, expr_cnt);
- for(i=0; i0)
- {
- new_bool_matcher=bool_matcher_new(exprs, expr_cnt, &mem_usage);
- if(!new_bool_matcher)
- {
- ret=-1;
- }
- }
- old_bool_matcher=bool_plugin_rt->matcher;
- bool_plugin_rt->matcher=new_bool_matcher;
- Maat_garbage_bagging(table_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))bool_matcher_free);
- EX_data_rt_update_commit(bool_plugin_rt->ex_data_rt);
- table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(bool_plugin_rt->ex_data_rt);
-
- free(exprs);
- free(exc_array);
- table_rt->fqdn_plugin.changed_flag=0;
- return ret;
-}
-
-int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
-{
- struct bool_expr_match results[size];
- int n_result=0;
- if(table_rt->table_type!=TABLE_TYPE_BOOL_PLUGIN)
- {
- return -1;
- }
- if(!table_rt->bool_plugin.matcher)
- {
- return 0;
- }
- n_item=ull_dedup(item_ids, n_item);
-
- n_result=bool_matcher_match(table_rt->bool_plugin.matcher, item_ids, n_item, results, size);
- for(int i=0; ibool_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
- }
- return n_result;
-}
-int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
-{
- long i=0,data_size=0;
- int ret=0;
- GIE_digest_t* digest_rule=NULL;
- GIE_digest_t** update_array=NULL;
- UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
-
-
- GIE_create_para_t para;
- para.gram_value=7;
- para.position_accuracy=10;
-
- const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q);
- if(q_cnt==0)
- {
- return 0;
- }
- if(table_rt->similar.gie_handle==NULL)
- {
- if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
- {
- para.ED_reexamine=1;
- para.format=GIE_INPUT_FORMAT_PLAIN;
- }
- else
- {
- para.ED_reexamine=0;
- para.format=GIE_INPUT_FORMAT_SFH;
- }
- table_rt->similar.gie_handle=GIE_create(¶);
- }
-
- update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
- for(i=0;isimilar.update_q, &digest_rule, &data_size);
- assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
- update_array[i]=digest_rule;
- digest_rule=NULL;
- }
- ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt);
- for(i=0;ioperation==GIE_INSERT_OPT)
- {
- table_rt->origin_rule_num++;
- }
- else
- {
- table_rt->origin_rule_num--;
- }
- destroy_digest_rule(update_array[i]);
- update_array[i]=NULL;
- }
- free(update_array);
- update_array=NULL;
-
- if(ret!=(int)q_cnt)
- {
- return -1;
- }
- return q_cnt;
-}
-
-int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt)
-{
- struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL;
- size_t rule_cnt=0;
- size_t i=0;
- struct ip_rule *rules=NULL;
- struct EX_data_container **exc_array=NULL;
- struct ip_plugin_runtime *ip_plugin=&(table_rt->ip_plugin);
- int ret=0;
- assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
- if(!ip_plugin->changed_flag)
- {
- return ret;
- }
- rule_cnt=EX_data_rt_list_updating_ex_containers(ip_plugin->ex_data_rt, &exc_array);
- rules=ALLOC(struct ip_rule, rule_cnt);
- for(i=0; i0)
- {
- new_ip_matcher=ip_matcher_new(rules, rule_cnt, &ip_plugin->mem_use_by_ip_matcher);
- if(!new_ip_matcher)
- {
- ret=-1;
- }
- }
- old_ip_matcher=ip_plugin->ip_matcher;
- ip_plugin->ip_matcher=new_ip_matcher;
- Maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
- EX_data_rt_update_commit(ip_plugin->ex_data_rt);
- table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(ip_plugin->ex_data_rt);
-
- free(rules);
- free(exc_array);
- exc_array=NULL;
- ip_plugin->changed_flag=0;
- return ret;
-}
-
-
-void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
-{
- struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
- struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
- size_t is_valid_offset=0, valid_len=0;
- size_t row_id_offset=0, row_id_len=0;
- struct ip_rule* ip_rule=NULL;
- int ret=0;
- if(ip_plugin_schema->ex_schema.set_flag)
- {
- ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "ip_plugin EX data process error: cannot find is_valid column %d of %s",
- ip_plugin_schema->row_id_column, row);
- return;
- }
- ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "ip_plugin EX data process error: cannot find row id column %d of %s",
- ip_plugin_schema->row_id_column, row);
- return;
- }
- ip_rule=ip_plugin_row2ip_rule(ip_plugin_schema, row);
- if(ip_rule==NULL)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "ip_plugin read ip error: %s", row);
- return;
-
- }
- if(atoi(row+is_valid_offset)==1)//add
- {
- ret=EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger);
- if(ret<0)
- {
- ip_rule_free(ip_rule);
- ip_rule=NULL;
- }
- }
- else
- {
-
- ret=EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
- ip_rule_free(ip_rule);
- ip_rule=NULL;
- }
- }
- else
- {
- EX_data_rt_cache_row_put(ip_plugin_rt->ex_data_rt, row);
- table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(ip_plugin_rt->ex_data_rt);
- }
- ip_plugin_rt->changed_flag=1;
- return;
-}
-int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
-{
- struct scan_result results[size];
- int n_result=0, i=0;
- if(!table_rt->ip_plugin.ip_matcher)
- {
- return 0;
- }
- n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size);
- for(i=0; iip_plugin.ex_data_rt, (struct EX_data_container *)results[i].tag);
- }
- return n_result;
-}
-
-
diff --git a/src/entry/Maat_table_schema.cpp b/src/entry/Maat_table_schema.cpp
deleted file mode 100644
index 60191a1..0000000
--- a/src/entry/Maat_table_schema.cpp
+++ /dev/null
@@ -1,1256 +0,0 @@
-#include "Maat_table_schema.h"
-#include "map_str2int.h"
-#include "Maat_utils.h"
-#include "cJSON.h"
-
-#include
-#include
-#include
-#include
-#include
-
-#define MAX_TABLE_NUM 256
-
-struct Maat_table_manager
-{
- struct Maat_table_schema* p_table_info[MAX_TABLE_NUM];
- size_t table_cnt;
- struct maat_kv_store* map_tablename2id;
- int active_plugin_table_num;
- int is_last_plugin_table_updating;
- void* logger;
-};
-enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type)
-{
- enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID;
- switch(table_type)
- {
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- case TABLE_TYPE_SIMILARITY:
- case TABLE_TYPE_DIGEST:
- ret=SCAN_TYPE_STRING;
- break;
- case TABLE_TYPE_INTERVAL:
- case TABLE_TYPE_INTERVAL_PLUS:
- ret=SCAN_TYPE_INTERVAL;
- break;
- case TABLE_TYPE_IP:
- case TABLE_TYPE_IP_PLUS:
- case TABLE_TYPE_COMPOSITION:
- ret=SCAN_TYPE_IP;
- break;
- case TABLE_TYPE_PLUGIN:
- ret=SCAN_TYPE_PLUGIN;
- break;
- case TABLE_TYPE_IP_PLUGIN:
- ret=SCAN_TYPE_IP;
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- ret=SCAN_TYPE_FQDN_PLUGIN;
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- ret=SCAN_TYPE_BOOL_PLUGIN;
- break;
- case TABLE_TYPE_COMPILE:
- ret=SCAN_TYPE_NONE;
- break;
- default:
- break;
- }
- return ret;
-}
-
-int read_expr_table_info(const char* line, struct Maat_table_schema* table, struct maat_kv_store* string2int_map)
-{
- int j=0, ret[4]={0};
- char table_type[16], src_charset[256], dst_charset[256], merge[4], quick_str_scan[32]={0};
- char *token=NULL, *sub_token=NULL, *saveptr;
- struct string_table_schema* p=&(table->expr);
- sscanf(line, "%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id),
- table->table_name[0],
- table_type,
- src_charset,
- dst_charset,
- merge,
- &(p->cross_cache_size),
- quick_str_scan);
- memset(ret, 0, sizeof(ret));
-
- ret[0]=maat_kv_read(string2int_map, table_type, (int*)&(table->table_type));
- ret[1]=maat_kv_read(string2int_map, src_charset, (int*)&(p->src_charset));
- ret[2]=maat_kv_read(string2int_map, merge, &(p->do_charset_merge));
- if(strlen(quick_str_scan)>0)
- {
- ret[3]=maat_kv_read(string2int_map, quick_str_scan, &(p->quick_expr_switch));
- }
- memset(quick_str_scan, 0, sizeof(quick_str_scan));
-
- for(j=0; j<4; j++)
- {
- if(ret[j]<0)
- {
- return -1;
- }
- }
- j=0;
- for (token = dst_charset; ; token= NULL)
- {
- sub_token= strtok_r(token, "/", &saveptr);
- if (sub_token == NULL)
- break;
- ret[3]=maat_kv_read(string2int_map, sub_token, (int*)&(p->dst_charset[j]));
- if(ret[3]>0)
- {
- if(p->dst_charset[j]==p->src_charset)
- {
- p->src_charset_in_dst=TRUE;
- }
- j++;
- }
- else
- {
- return -1;
- }
-
- }
- return 0;
-}
-
-Maat_table_schema* table_info_new(void)
-{
- struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1);
- p->conj_cnt=1;
- return p;
-}
-void table_info_free(struct Maat_table_schema*p)
-{
- free(p);
- return;
-}
-int _read_integer_arrary(char* string, int *array, int size)
-{
- char *token=NULL,*sub_token=NULL,*saveptr;
- int i=0;
- for (token = string, i=0; itype!=cJSON_Array)
- {
- goto error_out;
- }
- cJSON_ArrayForEach(tmp, json)
- {
- if(tmp->type!=cJSON_String)
- {
- goto error_out;
- }
- ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id);
- if(ret<0)
- {
- goto error_out;
- }
- physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
- physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
- if(physical_table_scan_typevirtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
- }
- }
- else //For compatible non-json physical description
- {
- ret=maat_kv_read(table_mgr->map_tablename2id, json_str, &tmp_table_id);
- if(ret<0)
- {
- goto error_out;
- }
- physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
- physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
- table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
- }
- cJSON_Delete(json);
- free(copy_line);
- return 0;
-
-error_out:
- if(json) cJSON_Delete(json);
- free(copy_line);
- return -1;
-
-}
-
-int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
-{
- int i=0,ret=0;
- size_t offset=0, len=0;
- cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
- char* copy_line=NULL, *plug_info=NULL;
- struct plugin_table_schema* plugin_desc=&(p->plugin);
- copy_line=_maat_strdup(line);
- ret=get_column_pos(copy_line, COLUMN_PLUGIN_SCHEMA_JSON, &offset, &len);
- if(ret<0)
- {
- goto error_out;
- }
- if(offset+lenvalid_flag_column));
- if(ret==0||ret==EOF)
- {
- plugin_desc->valid_flag_column=-1;
- }
- free(copy_line);
- return 0;
- }
- json=cJSON_Parse(plug_info);
- if(!json)
- {
- goto error_out;
- }
- tmp=cJSON_GetObjectItem(json, "key");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- plugin_desc->key_column=tmp->valueint;
- }
- tmp=cJSON_GetObjectItem(json, "valid");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- plugin_desc->valid_flag_column=tmp->valueint;
- }
- tmp=cJSON_GetObjectItem(json, "tag");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- plugin_desc->rule_tag_column=tmp->valueint;
- }
- tmp=cJSON_GetObjectItem(json, "foreign");
- if(tmp!=NULL)
- {
- if(tmp->type==cJSON_String)
- {
- plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM);
- }
- else if(tmp->type==cJSON_Array)
- {
- plugin_desc->n_foreign= cJSON_GetArraySize(tmp);
- for(i=0;in_foreign; i++)
- {
- array_item=cJSON_GetArrayItem(tmp, i);
- assert(array_item->type==cJSON_Number);
- plugin_desc->foreign_columns[i]=array_item->valueint;
- }
- }
- }
- cJSON_Delete(json);
-
- free(copy_line);
- return 0;
-error_out:
- free(copy_line);
- return -1;
-}
-int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
-{
- int ret=0, read_cnt=0;
- size_t offset=0, len=0;
- cJSON* json=NULL, *tmp=NULL;
- char* copy_line=NULL, *ip_plugin_info=NULL;
- struct ip_plugin_table_schema* ip_plugin_schema=&(p->ip_plugin);
- copy_line=_maat_strdup(line);
- ret=get_column_pos(copy_line, COLUMN_IP_PLUGIN_SCHEMA_JSON, &offset, &len);
- if(ret<0)
- {
- goto error_out;
- }
- if(offset+lentype==cJSON_Number)
- {
- ip_plugin_schema->row_id_column=tmp->valueint;
- read_cnt++;
- }
-
- tmp=cJSON_GetObjectItem(json, "ip_type");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- ip_plugin_schema->ip_type_column=tmp->valueint;
- read_cnt++;
- }
- tmp=cJSON_GetObjectItem(json, "start_ip");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- ip_plugin_schema->start_ip_column=tmp->valueint;
- read_cnt++;
- }
- tmp=cJSON_GetObjectItem(json, "end_ip");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- ip_plugin_schema->end_ip_column=tmp->valueint;
- read_cnt++;
- }
-
- tmp=cJSON_GetObjectItem(json, "valid");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- ip_plugin_schema->valid_flag_column=tmp->valueint;
- read_cnt++;
- }
- ip_plugin_schema->rule_tag_column=-1;
- tmp=cJSON_GetObjectItem(json, "tag");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- ip_plugin_schema->rule_tag_column=tmp->valueint;
- //read_cnt++; Tag is optional, so NOT ++ intentionally.
- }
-
- cJSON_Delete(json);
-
- free(copy_line);
- if(read_cnt<5)
- {
- return -1;
- }
- else
- {
- return 0;
- }
-error_out:
- free(copy_line);
- return -1;
-
-}
-int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p)
-{
- int ret=0, read_cnt=0;
- size_t offset=0, len=0;
- cJSON* json=NULL, *tmp=NULL;
- char* copy_line=NULL, *fqnd_plugin_schema_json=NULL;
- struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin);
-
- copy_line=_maat_strdup(line);
- ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SCHEMA_JSON, &offset, &len);
- if(ret<0)
- {
- goto error_out;
- }
- if(offset+lentype==cJSON_Number)
- {
- fqdn_plugin_schema->row_id_column=tmp->valueint;
- read_cnt++;
- }
-
- tmp=cJSON_GetObjectItem(json, "is_suffix_match");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint;
- read_cnt++;
- }
- tmp=cJSON_GetObjectItem(json, "fqdn");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- fqdn_plugin_schema->fqdn_column=tmp->valueint;
- read_cnt++;
- }
-
- tmp=cJSON_GetObjectItem(json, "valid");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- fqdn_plugin_schema->valid_flag_column=tmp->valueint;
- read_cnt++;
- }
- fqdn_plugin_schema->rule_tag_column=-1;
- tmp=cJSON_GetObjectItem(json, "tag");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- fqdn_plugin_schema->rule_tag_column=tmp->valueint;
- //read_cnt++; Tag is optional, so NOT ++ intentionally.
- }
-
- cJSON_Delete(json);
-
- free(copy_line);
- if(read_cnt<4)
- {
- return -1;
- }
- else
- {
- return 0;
- }
-error_out:
- free(copy_line);
- return -1;
-
-}
-int read_bool_plugin_table_schema(const char* line, struct Maat_table_schema* p)
-{
- int ret=0, read_cnt=0;
- size_t offset=0, len=0;
- cJSON* json=NULL, *tmp=NULL;
- char* copy_line=NULL, *schema_json=NULL;
- struct bool_plugin_table_schema* bool_plugin_schema=&(p->bool_plugin);
-
- copy_line=_maat_strdup(line);
- ret=get_column_pos(copy_line, COLUMN_BOOL_PLUGIN_SCHEMA_JSON, &offset, &len);
- if(ret<0)
- {
- goto error_out;
- }
- if(offset+lentype==cJSON_Number)
- {
- bool_plugin_schema->row_id_column=tmp->valueint;
- read_cnt++;
- }
- tmp=cJSON_GetObjectItem(json, "bool_expr");
- if(tmp!=NULL && tmp->type==cJSON_Number)
- {
- bool_plugin_schema->bool_expr_column=tmp->valueint;
- read_cnt++;
- }
-
- tmp=cJSON_GetObjectItem(json, "valid");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- bool_plugin_schema->valid_flag_column=tmp->valueint;
- read_cnt++;
- }
- bool_plugin_schema->rule_tag_column=-1;
- tmp=cJSON_GetObjectItem(json, "tag");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- bool_plugin_schema->rule_tag_column=tmp->valueint;
- //read_cnt++; Tag is optional, so NOT ++ intentionally.
- }
-
- cJSON_Delete(json);
-
- free(copy_line);
- if(read_cnt<3)
- {
- return -1;
- }
- else
- {
- return 0;
- }
-error_out:
- free(copy_line);
- return -1;
-
-}
-
-int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
-{
- int ret=0;
- size_t offset=0, len=0;
- cJSON* json=NULL, *tmp=NULL;
- char* copy_line=NULL, *composition_info=NULL;
- struct composition_table_schema* composition_schema=&(p->composition);
- copy_line=_maat_strdup(line);
- ret=get_column_pos(copy_line, COLUMN_COMPOSITION_SCHEMA_JSON, &offset, &len);
- if(ret<0)
- {
- goto error_out;
- }
- if(offset+lentype==cJSON_String)
- {
- ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP]));
- if(ret<0)
- {
- MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- tmp->valuestring,
- p->table_name[0],
- p->table_id);
- goto error_out;
- }
- }
- tmp=cJSON_GetObjectItem(json, "destination");
- if(tmp!=NULL && tmp->type==cJSON_String)
- {
- ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP]));
- if(ret<0)
- {
- MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- tmp->valuestring,
- p->table_name[0],
- p->table_id);
- goto error_out;
- }
-
- }
- tmp=cJSON_GetObjectItem(json, "session");
- if(tmp!=NULL && tmp->type==cJSON_String)
- {
- ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]));
- if(ret<0)
- {
- MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- tmp->valuestring,
- p->table_name[0],
- p->table_id);
- goto error_out;
- }
- }
- cJSON_Delete(json);
- free(copy_line);
- return 0;
-
-error_out:
- free(copy_line);
- return -1;
-
-}
-void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr)
-{
- size_t i=0;
- for(i=0;ip_table_info[i]==NULL)
- {
- continue;
- }
- table_info_free(table_mgr->p_table_info[i]);
- table_mgr->p_table_info[i]=NULL;
- }
- maat_kv_store_free(table_mgr->map_tablename2id);
- free(table_mgr);
- return;
-}
-
-struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger)
-{
- struct Maat_table_manager* table_mgr=NULL;
- FILE*fp=NULL;
- char line[MAX_TABLE_LINE_SIZE];
- int i=0, ret=0;
- char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
- struct maat_kv_store* reserved_word_map=NULL;;
- struct Maat_table_schema*p=NULL;
- struct Maat_table_schema*conj_table=NULL;
- fp=fopen(table_info_path,"r");
- if(fp==NULL)
- {
- fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno));
- return NULL;
- }
- table_mgr=ALLOC(struct Maat_table_manager, 1);
- struct Maat_table_schema** p_table_info=table_mgr->p_table_info;
- size_t n_table=MAX_TABLE_NUM;
- table_mgr->logger=logger;
- table_mgr->map_tablename2id=maat_kv_store_new();
-
- reserved_word_map=maat_kv_store_new();
- maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
- maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP);
- maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
- maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE);
- maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
- maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
- maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
- maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
- maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
- maat_kv_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL);
- maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
- maat_kv_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS);
- maat_kv_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST);
- maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
- maat_kv_register(reserved_word_map, "group", TABLE_TYPE_GROUP);
- maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
- maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
- maat_kv_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY);
- maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
- maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION);
- maat_kv_register(reserved_word_map, "quickoff", 0);
- maat_kv_register(reserved_word_map, "quickon", 1);
- maat_kv_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE);
-// maat_kv_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
-
- const char** charset_name_list=charset_get_all_name();
- for(i=0;i0)
- {
- maat_kv_register(reserved_word_map, charset_name_list[i], i);
- }
- else
- {
- break;
- }
- }
-
- maat_kv_register(reserved_word_map,"yes", 1);
- maat_kv_register(reserved_word_map,"no", 0);
-
-
- i=0;
- while(NULL!=fgets(line,sizeof(line),fp))
- {
- i++;
-
- if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4)
- {
- continue;
- }
- p=table_info_new();
-
- ret=sscanf(line, "%d\t%s\t%s\t%[a-z0-9\t ]", &(p->table_id),
- p->table_name[0],
- table_type_str,
- not_care);
- if(ret<3)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: not enough column.",table_info_path,i);
- continue;
- }
- ret=maat_kv_read(reserved_word_map, table_type_str, (int*)&(p->table_type));
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error:invalid table type.",table_info_path,i);
- goto invalid_table;
- }
- switch(p->table_type)
- {
- case TABLE_TYPE_EXPR:
- case TABLE_TYPE_EXPR_PLUS:
- ret=read_expr_table_info(line, p, reserved_word_map);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal column.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error:illegal column.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_PLUGIN:
- ret=read_plugin_table_schema(line, p);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal plugin table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error: illegal plugin table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_IP_PLUGIN:
- ret=read_ip_plugin_table_schema(line, p);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal ip_plugin table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: illegal ip_plugin table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- ret=read_fqdn_plugin_table_schema(line, p);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal fqdn_plugin table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: illegal fqdn_plugin table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- ret=read_bool_plugin_table_schema(line, p);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal bool_plugin table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: illegal bool_plugin table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_COMPOSITION:
- ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal composition table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: illegal composition table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_VIRTUAL:
- ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map);
- if(ret<0)
- {
- fprintf(stderr, "Maat read table info %s line %d error: illegal virtual table schema.\n", table_info_path, i);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: illegal virtual table schema.", table_info_path, i);
- goto invalid_table;
- }
- break;
- case TABLE_TYPE_COMPILE:
- ret=sscanf(not_care,"%[a-z0-9]", tmp_str);
- if(ret>0)
- {
- ret=maat_kv_read(reserved_word_map, tmp_str, (int*)&(p->compile.user_region_encoding));
- }
- if(ret!=1)
- {
- p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
- }
- break;
- default:
- break;
- }
-
- if((unsigned int)p->table_id>=n_table)
- {
- fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table);
-
- goto invalid_table;
- }
- ret=maat_kv_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Duplicate table %s of table id %d",
- p->table_name[0],
- p->table_id);
- goto invalid_table;
- }
- if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
- {
- conj_table=p_table_info[p->table_id];
- if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Maat read table info %s line %d error: reach tableid %d conjunction upper limit."
- ,table_info_path,i,p->table_id);
- goto invalid_table;
- }
- memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
- conj_table->conj_cnt++;
- MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
- "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
- ,table_info_path,i,p->table_name[0]
- ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
- //use goto to free the conjunctioned table_info
- goto invalid_table;
- }
- p_table_info[p->table_id]=p;
- table_mgr->table_cnt++;
- continue;
-
-invalid_table:
- table_info_free(p);
- p=NULL;
- }
- fclose(fp);
- maat_kv_store_free(reserved_word_map);
- return table_mgr;
-}
-size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr)
-{
- return MAX_TABLE_NUM;
-}
-size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr)
-{
- return table_mgr->table_cnt;
-}
-int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
-{
- int i=0;
- for(i=0; i< MAX_TABLE_NUM; i++)
- {
- if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE)
- {
- strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
- return 1;
- }
- }
- return 0;
-}
-int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
-{
- int i=0;
- for(i=0; i< MAX_TABLE_NUM; i++)
- {
- if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2COMPILE)
- {
- strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
- return 1;
- }
- }
- return 0;
-}
-int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
-{
- int i=0;
- for(i=0; i< MAX_TABLE_NUM; i++)
- {
- if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2GROUP)
- {
- strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
- return 1;
- }
- }
- return 0;
-}
-enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id)
-{
- if(table_id>MAX_TABLE_NUM)
- {
- return TABLE_TYPE_INVALID;
- }
- if(table_mgr->p_table_info[table_id])
- {
- return table_mgr->p_table_info[table_id]->table_type;
- }
- return TABLE_TYPE_INVALID;
-
-}
-
-struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id)
-{
- enum MAAT_SCAN_TYPE tab_scan_type;
- struct Maat_table_schema **p_table_info=table_mgr->p_table_info;
- size_t n_table=MAX_TABLE_NUM;
-
- struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL;
- if((unsigned int) table_id>n_table)
- {
- return NULL;
- }
- if(p_table_info[table_id]==NULL)
- {
- return NULL;
- }
- p_table=p_table_info[table_id];
- if(p_table==NULL)
- {
- return NULL;
- }
- if(p_table->table_type==TABLE_TYPE_VIRTUAL)
- {
- p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]];
- *virutal_table_id=table_id;
- }
- else
- {
- p_physical_table=p_table;
- if(virutal_table_id) *virutal_table_id=0;
- }
- tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type);
- if(tab_scan_type!=scan_type)
- {
- return NULL;
- }
- return p_physical_table;
-}
-int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
-{
- int table_id=-1,ret=0;
- ret=maat_kv_read(table_mgr->map_tablename2id, table_name, &table_id);
- if(ret>0)
- {
- return table_id;
- }
- else
- {
- return -1;
- }
-}
-int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr,
- int table_id,
- Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
- Maat_update_callback_t *update,//table line ,u_para
- Maat_finish_callback_t *finish,//u_para
- void* u_para)
-{
- int idx=0;
- struct Maat_table_schema *p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
- struct plugin_table_schema *plugin_desc=&(p_table->plugin);
- if(p_table==NULL)
- {
- return -1;
- }
-
- idx=plugin_desc->cb_plug_cnt;
- if(idx==MAX_PLUGIN_PER_TABLE)
- {
- return -1;
- }
- plugin_desc->cb_plug_cnt++;
- plugin_desc->cb_plug[idx].start=start;
- plugin_desc->cb_plug[idx].update=update;
- plugin_desc->cb_plug[idx].finish=finish;
- plugin_desc->cb_plug[idx].u_para=u_para;
- return 1;
-}
-
-struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx)
-{
- int table_id=-1;
- struct Maat_table_schema *p_table=NULL;
-
- table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name);
- if(table_id<0)
- {
- return NULL;
- }
- p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
- if(!p_table)
- {
- return NULL;
- }
- if(idxcompile.ex_data_num)
- {
- return p_table->compile.ex_desc+idx;
- }
- return NULL;
-
-}
-int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
- Maat_rule_EX_new_func_t *new_func,
- Maat_rule_EX_free_func_t* free_func,
- Maat_rule_EX_dup_func_t* dup_func,
- long argl, void *argp)
-{
- int table_id=-1;
- struct Maat_table_schema *p_table=NULL;
- table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name);
- if(table_id<0)
- {
- return -1;
- }
- p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
- if(!p_table)
- {
- return -1;
- }
- int idx=-1;
-
- struct compile_table_schema* compile_desc=&(p_table->compile);
- if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
- {
- return -1;
- }
- idx=compile_desc->ex_data_num;
- compile_desc->ex_desc[idx].idx=idx;
- compile_desc->ex_desc[idx].table_id=table_id;
- compile_desc->ex_desc[idx].argl=argl;
- compile_desc->ex_desc[idx].argp=argp;
- compile_desc->ex_desc[idx].new_func=new_func;
- compile_desc->ex_desc[idx].free_func=free_func;
- compile_desc->ex_desc[idx].dup_func=dup_func;
-
- compile_desc->ex_data_num++;
-
- return idx;
-}
-struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema)
-{
- struct EX_data_schema *ex_schema=NULL;
- switch(table_schema->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- ex_schema=&table_schema->plugin.ex_schema;
- break;
- case TABLE_TYPE_IP_PLUGIN:
- ex_schema=&table_schema->ip_plugin.ex_schema;
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- ex_schema=&table_schema->fqdn_plugin.ex_schema;
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- ex_schema=&table_schema->bool_plugin.ex_schema;
- break;
- default:
- break;
- }
- return ex_schema;
-}
-
-int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema,
- Maat_plugin_EX_new_func_t* new_func,
- Maat_plugin_EX_free_func_t* free_func,
- Maat_plugin_EX_dup_func_t* dup_func,
- Maat_plugin_EX_key2index_func_t* key2index_func,
- long argl, void *argp,
- void* logger)
-{
- struct EX_data_schema *ex_schema=NULL;
- if(new_func==NULL || free_func==NULL || dup_func==NULL )
- {
- assert(0);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
- return -1;
- }
- ex_schema=Maat_table_schema_get_EX_data_schema(table_schema);
- if(ex_schema==NULL)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a valid plugin table.", __FUNCTION__);
- return -1;
- }
- if(ex_schema->set_flag)
- {
- assert(0);
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data schema already registed.", __FUNCTION__);
- return -1;
- }
- ex_schema->new_func=new_func;
- ex_schema->free_func=free_func;
- ex_schema->dup_func=dup_func;
- ex_schema->key2index_func=key2index_func;//Set but not used.
- ex_schema->argl=argl;
- ex_schema->argp=argp;
- ex_schema->set_flag=1;
- return 0;
-
-}
-void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
-{
- table_mgr->active_plugin_table_num=0;
- int i=0, j=0;
- struct Maat_table_schema* p_table=NULL;
- struct plugin_table_schema* plugin_desc=NULL;
-
- for(i=0; ip_table_info[i];
- plugin_desc=&(p_table->plugin);
- if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0)
- {
- continue;
- }
-
- table_mgr->active_plugin_table_num++;
-
- for(j=0;jcb_plug_cnt;j++)
- {
- if(plugin_desc->cb_plug[j].start!=NULL)
- {
- plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para);
- }
- }
- }
-
-}
-void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr)
-{
- int i=0, j=0;
- struct Maat_table_schema* p_table=NULL;
- struct plugin_table_schema* plugin_desc=NULL;
-
- int call_plugin_table_cnt=0;
- for(i=0;ip_table_info[i];
- if(p_table==NULL)
- {
- continue;
- }
- switch(p_table->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- plugin_desc=&(p_table->plugin);
- call_plugin_table_cnt++;
- if(call_plugin_table_cnt==table_mgr->active_plugin_table_num)
- {
- table_mgr->is_last_plugin_table_updating=1;
- }
- for(j=0;jcb_plug_cnt;j++)
- {
- if(plugin_desc->cb_plug[j].finish!=NULL)
- {
- plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para);
- }
- }
- table_mgr->is_last_plugin_table_updating=0;
- break;
- default:
- break;
- }
- }
- table_mgr->active_plugin_table_num=0;
- return;
-}
-int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr)
-{
- return table_mgr->is_last_plugin_table_updating;
-}
-struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
-{
- struct Maat_table_schema * p_table=NULL;
- int table_id=0;
- table_id=Maat_table_manager_get_id_by_name(table_mgr, table_name);
- if(table_id<0)
- {
- return NULL;
- }
- p_table=table_mgr->p_table_info[table_id];
- return p_table;
-}
-void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name)
-{
- int i=0;
- for(i=0; iconj_cnt; i++)
- {
- if(0==strcmp(p_table->table_name[i], table_name))
- {
- p_table->updating_name=i;
- }
- }
- assert(i<=p_table->conj_cnt);
-}
-struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id)
-{
- if(table_id>MAX_TABLE_NUM||table_id<0)
- {
- return NULL;
- }
-
- return table_mgr->p_table_info[table_id];
-}
-
-int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type)
-{
- int ret=-1;
- struct Maat_table_schema* p_table=Maat_table_manager_get_by_id_raw(table_mgr, parent_table_id);
- if(p_table->table_type!=TABLE_TYPE_COMPOSITION)
- {
- return -1;
- }
- ret=p_table->composition.component_table_id[type];
- return ret;
-}
-int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table)
-{
- int valid_flag_column=-1;
- switch(p_table->table_type)
- {
- case TABLE_TYPE_PLUGIN:
- valid_flag_column=p_table->plugin.valid_flag_column;
- break;
- case TABLE_TYPE_IP_PLUGIN:
- valid_flag_column=p_table->ip_plugin.valid_flag_column;
- break;
- case TABLE_TYPE_FQDN_PLUGIN:
- valid_flag_column=p_table->fqdn_plugin.valid_flag_column;
- break;
- case TABLE_TYPE_BOOL_PLUGIN:
- valid_flag_column=p_table->bool_plugin.valid_flag_column;
- break;
- default:
- valid_flag_column=-1;
- break;
- }
- return valid_flag_column;
-}
-
diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp
deleted file mode 100644
index 7195ce4..0000000
--- a/src/entry/Maat_utils.cpp
+++ /dev/null
@@ -1,594 +0,0 @@
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include "Maat_utils.h"
-pid_t gettid()
-{
- return syscall(SYS_gettid);
-}
-const char* module_name_str(const char*name)
-{
- static __thread char module[64];
- snprintf(module,sizeof(module),"%s(%d)", name, gettid());
- return module;
-}
-
-int converHextoint(char srctmp)
-{
- if(isdigit(srctmp))
- {
- return srctmp-'0';
- }
- else
- {
- char temp=toupper(srctmp);
- temp=temp-'A'+10;
- return temp;
- }
-}
-int hex2bin(char *hex,int hex_len,char *binary,int size)
-{
- int i=0;
- int resultlen=0;
- int high,low;
- for(i=0;iresultlen; i+=2,resultlen++)
- {
- high=converHextoint(hex[i]);
- low=converHextoint(hex[i+1]);
- binary[resultlen]=high*16+low;
- }
- size=resultlen;
- binary[resultlen]='\0';
- return resultlen;
-}
-//functioned as strdup, for dictator compatible.
-char* _maat_strdup(const char* s)
-{
- char*d=NULL;
- if(s==NULL)
- {
- return NULL;
- }
- d=(char*)malloc(strlen(s)+1);
- memcpy(d,s,strlen(s)+1);
- return d;
-}
-char* str_tolower(char* string)
-{
- int i=0;
- for(i=0;i<(int)strlen(string);i++)
- {
- string[i]=(char)tolower(string[i]);
- }
- return string;
-}
-char * strchr_esc(char* s,const char delim)
-{
- char *token;
- if(s==NULL)
- return NULL;
- for(token=s;*token!='\0';token++)
- {
- if(*token=='\\')
- {
- token++;
- continue;
- }
- if(*token==delim)
- break;
- }
- if (*token == '\0')
- {
- return NULL;
- }
- else
- {
- return token;
- }
-}
-char *strtok_r_esc(char *s, const char delim, char **save_ptr)
-{
- char *token;
-
- if (s == NULL) s = *save_ptr;
-
- /* Scan leading delimiters. */
- token=strchr_esc(s,delim);
- if(token==NULL)
- {
- *save_ptr=token;
- return s;
- }
- /* Find the end of the token. */
- *token='\0';
- token++;
- *save_ptr=token;
-
- return s;
-}
-char *str_unescape_and(char*s)
-{
- int i=0,j=0;
- for(i=0,j=0;i<(int)strlen(s);i++)
- {
- if(s[i]=='\\'&&s[i+1]=='&')
- {
- s[j]='&';
- i++;
- j++;
- }
- else{
- s[j]=s[i];
- j++;
- }
- }
- s[j]='\0';
- return s;
-}
-char* str_unescape(char* s)
-{
- int i=0,j=0;
- int len=strlen(s);
- for(i=0,j=0;i %s", src_file, dst_file);
- return system(cmd);
-}
-
-int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password)
-{
- char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
- snprintf(cmd,sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s", password, src_file, dst_file);
- return system(cmd);
-}
-int system_cmd_rm(const char* src_file)
-{
- char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
- snprintf(cmd,sizeof(cmd), "rm %s -f", src_file);
- return system(cmd);
-}
-char* md5_file(const char* filename, char* md5string)
-{
- FILE* fp=NULL;
- int i=0;
- unsigned char md5[MD5_DIGEST_LENGTH];
- struct stat file_info;
- stat(filename, &file_info);
- size_t file_size=file_info.st_size;
-
- fp=fopen(filename,"r");
- if(fp==NULL)
- {
- return NULL;
- }
- char* file_buff=(char*)malloc(file_size);
- fread(file_buff,1,file_size,fp);
- fclose(fp);
-
- MD5((const unsigned char *)(file_buff), (unsigned long)(file_size), md5);
- for(i = 0; i < MD5_DIGEST_LENGTH; ++i)
- {
- sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]);
- }
- free(file_buff);
- return md5string;
-}
-const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
- "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8", "windows-1251", ""};
-
-const char** charset_get_all_name(void)
-{
- return CHARSET_STRING;
-}
-const char* charset_get_name(enum MAAT_CHARSET charset)
-{
- return CHARSET_STRING[charset];
-}
-int lqueue_destroy_cb(void *data, long data_len, void *arg)
-{
- assert(0);
- return 0;
-}
-
-int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz)
-{
- int ret=0, out_blk_len=0;
- int out_buff_len=0, out_buff_offset=0;
- EVP_CIPHER_CTX *ctx;
-
- unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
- unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
- memset(cipher_key,0,sizeof(cipher_key));
- memset(cipher_iv,0,sizeof(cipher_iv));
-
- const EVP_CIPHER *cipher;
- const EVP_MD *dgst=NULL;
- const unsigned char *salt=NULL;
-
- OpenSSL_add_all_algorithms();
- cipher=EVP_get_cipherbyname(algorithm);
- if(cipher==NULL)
- {
- snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm);
- return 0;
- }
- dgst=EVP_get_digestbyname("md5");
- if(dgst==NULL)
- {
- snprintf(err_str, err_str_sz, "Get MD5 object failed.");
- return 0;
- }
- ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv);
- if(ret==0)
- {
- snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
- return 0;
- }
- /* Don't set key or IV right away; we want to check lengths */
- ctx = EVP_CIPHER_CTX_new();
- EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt);
- OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0);
- OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
-
- /* Now we can set key and IV */
- //It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call).
- EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1);
- out_buff_len=inlen+EVP_CIPHER_block_size(cipher)-1;
- *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char));
- if (!EVP_CipherUpdate(ctx, *pp_out+out_buff_offset, &out_blk_len, inbuf, inlen))
- {
- snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed.");
- EVP_CIPHER_CTX_free(ctx);
- goto error_out;
- }
- out_buff_offset+=out_blk_len;
- if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len))
- {
- snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?");
- EVP_CIPHER_CTX_free(ctx);
- goto error_out;
- }
- out_buff_offset+=out_blk_len;
- EVP_CIPHER_CTX_free(ctx);
- *out_sz=out_buff_offset;
- return 0;
-
-error_out:
- free(*pp_out);
- *pp_out=NULL;
- return -1;
-}
-int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz)
-{
- int ret=0;
- FILE* fp=NULL;
- struct stat fstat_buf;
- size_t read_size=0;
-
- ret=stat(file_name, &fstat_buf);
- if(ret!=0)
- {
- return -1;
- }
- fp=fopen(file_name, "r");
- if(fp==NULL)
- {
- return -1;
- }
- *out_sz=fstat_buf.st_size;
- *pp_out=(unsigned char*)calloc(1, *out_sz+1);
- read_size=fread(*pp_out, 1, *out_sz, fp);
- if(read_size!= *out_sz)
- {
- free(*pp_out);
- pp_out=NULL;
- return -1;
- }
-
- fclose(fp);
- fp=NULL;
- return 0;
-}
-int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz)
-{
- int ret=0;
- size_t file_sz=0;
- unsigned char* file_buff=NULL;
- ret=load_file_to_memory(file_name, &file_buff, &file_sz);
- if(ret<0)
- {
- return -1;
- }
- ret=crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
- free(file_buff);
- file_buff=NULL;
- return ret;
-}
-
-int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data,
- size_t *out_uncompressed_sz)
-{
- z_stream strm;
- strm.zalloc = NULL;
- strm.zfree = NULL;
- strm.opaque = NULL;
-
- strm.avail_in = in_compressed_sz;
- strm.avail_out = *out_uncompressed_sz;
- strm.next_in = (Bytef*) in_compressed_data;
- strm.next_out = *out_uncompressed_data;
-
- int ret = -1;
- ret = inflateInit2(&strm, MAX_WBITS+16);
- if (ret == Z_OK)
- {
- ret = inflate(&strm, Z_FINISH);
- if (ret == Z_STREAM_END)
- {
- *out_uncompressed_sz = strm.total_out;
- ret = inflateEnd(&strm);
- return ret;
- }
- }
- inflateEnd(&strm);
- return ret;
-}
-int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data,
- size_t *out_uncompressed_sz)
-{
-
-
- int z_result;
- int ret=-1;
- size_t buffer_sz=in_compressed_sz*2;
- *out_uncompressed_data = (unsigned char*) malloc(buffer_sz);
- do{
- *out_uncompressed_sz=buffer_sz;
- z_result = gzip_uncompress_one_try(
- in_compressed_data,
- in_compressed_sz,
- out_uncompressed_data,
- out_uncompressed_sz);
-
- switch( z_result )
- {
- case Z_OK:
- ret=0;
- break;
- case Z_BUF_ERROR:
- buffer_sz*=2;
- *out_uncompressed_data=(unsigned char*) realloc(*out_uncompressed_data, buffer_sz);
- break;
- default:
- ret=-1;
- break;
- }
-
- }while(z_result==Z_BUF_ERROR);
- return ret;
-}
-
-
-enum MAAT_IP_FORMAT ip_format_str2int(const char* format)
-{
- if(0==strcasecmp(format, "range"))
- {
- return FORMAT_RANGE;
- }
- else if(0==strcasecmp(format, "mask"))
- {
- return FORMAT_MASK;
- }
- else if(0==strcasecmp(format, "CIDR"))
- {
- return FORMAT_CIDR;
- }
- else
- {
- assert(0);
- }
- return FORMAT_UNKNOWN;
-}
-int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[])
-{
- unsigned int ipv4_addr=0, ipv4_mask=0, ipv4_range_end=0;
- unsigned int ipv6_addr[4]={0}, ipv6_mask[4]={0}, ipv6_range_end[4]={0};
- int cidr=0, bit32=0;
- int ret=0, i=0;
- if(ip_type!=4 && ip_type!=6)
- {
- assert(0);
- return -1;
- }
- if(ip_type==4)
- {
- ret=inet_pton(AF_INET, ip1, &ipv4_addr);
- if(ret<=0)
- {
- return -1;
- }
- ipv4_addr=ntohl(ipv4_addr);
- switch (format)
- {
- case FORMAT_RANGE:
- range_begin[0]=ipv4_addr;
- ret=inet_pton(AF_INET, ip2, &ipv4_range_end);
- if(ret<=0)
- {
- return -1;
- }
- ipv4_range_end=ntohl(ipv4_range_end);
- range_end[0]=ipv4_range_end;
- break;
- case FORMAT_MASK:
- ret=inet_pton(AF_INET, ip2, &ipv4_mask);
- if(ret<=0)
- {
- return -1;
- }
- ipv4_mask=ntohl(ipv4_mask);
- range_begin[0]=ipv4_addr&ipv4_mask;
- range_end[0]=ipv4_addr|~ipv4_mask;
- break;
- case FORMAT_CIDR:
- cidr=atoi(ip2);
- if(cidr>32||cidr<0)
- {
- return -1;
- }
- ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL;
- range_begin[0]=ipv4_addr&ipv4_mask;
- range_end[0]=ipv4_addr|~ipv4_mask;
- break;
- default:
- assert(0);
- }
- }
- else //ipv6
- {
- ret=inet_pton(AF_INET6, ip1, ipv6_addr);
- if(ret<=0)
- {
- return -1;
- }
- ipv6_ntoh(ipv6_addr);
- switch(format)
- {
- case FORMAT_RANGE:
- ret=inet_pton(AF_INET6, ip2, ipv6_range_end);
- if(ret<=0)
- {
- return -1;
- }
- ipv6_ntoh(ipv6_range_end);
- memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr));
- memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end));
- break;
- case FORMAT_MASK:
- ret=inet_pton(AF_INET6, ip2, ipv6_mask);
- if(ret<=0)
- {
- return -1;
- }
- ipv6_ntoh(ipv6_mask);
- for(i=0; i<4; i++)
- {
- range_begin[i]=ipv6_addr[i]&ipv6_mask[i];
- range_end[i] = ipv6_addr[i]|~ipv6_mask[i];
- }
- break;
- case FORMAT_CIDR:
- cidr=atoi(ip2);
- if(cidr>128||cidr<0)
- {
- return -1;
- }
- for(i=0; i<4; i++)
- {
- bit32=128-cidr-32*(3-i);
- if(bit32<0) bit32=0;
- ipv6_mask[i]=(0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL;
- range_begin[i]=ipv6_addr[i]&ipv6_mask[i];
- range_end[i] = ipv6_addr[i]|~ipv6_mask[i];
- }
- break;
- default:
- assert(0);
- }
- }
- return 0;
-}
-
diff --git a/src/entry/UniversalBoolMatch.cpp b/src/entry/UniversalBoolMatch.cpp
deleted file mode 100644
index ff91d60..0000000
--- a/src/entry/UniversalBoolMatch.cpp
+++ /dev/null
@@ -1,260 +0,0 @@
-#include "UniversalBoolMatch.h"
-#include