From 7e860f2c581cca805afaff7af5812267020b6d26 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 26 Oct 2022 14:41:22 +0800 Subject: [PATCH] Refactor from scratch. --- .gitignore | 45 - .gitlab-ci.yml | 224 - CMakeLists.txt | 40 - Makefile | 41 - ancient_history.txt | 48 - autorevision.sh | 1268 - ci/get-nprocessors.sh | 48 - ci/perpare_pulp3_netrc.sh | 3 - ci/travis.sh | 72 - cmake/Package.cmake | 57 - cmake/Version.cmake | 54 - cmake/changelog.sh | 6 - inc/Maat_command.h | 221 - inc/Maat_rule.h | 353 - inc/bool_matcher.h | 64 - inc/gram_index_engine.h | 72 - inc/stream_fuzzy_hash.h | 78 - include/maat/maat.h | 23 + lib/.gitignore | 9 - readme.md | 25 - src/CMakeLists.txt | 46 - src/entry/.gitignore | 9 - src/entry/FQDN_engine.cpp | 352 - src/entry/Maat_api.cpp | 2667 - src/entry/Maat_command.cpp | 2482 - src/entry/Maat_ex_data.cpp | 397 - src/entry/Maat_garbage_collection.cpp | 96 - src/entry/Maat_hierarchy.cpp | 1550 - src/entry/Maat_rule.cpp | 2815 - src/entry/Maat_stat.cpp | 368 - src/entry/Maat_table_runtime.cpp | 980 - src/entry/Maat_table_schema.cpp | 1256 - src/entry/Maat_utils.cpp | 594 - src/entry/UniversalBoolMatch.cpp | 260 - src/entry/bool_matcher.cpp | 216 - src/entry/cJSON.c | 2932 - src/entry/config_monitor.cpp | 418 - src/entry/dynamic_array.cpp | 53 - src/entry/gram_index_engine.c | 1354 - src/entry/hyperscan_adapter.cpp | 67 - src/entry/interval_index.c | 737 - src/entry/json2iris.cpp | 1286 - src/entry/map_str2int.cpp | 124 - src/entry/rbtree.c | 548 - src/entry/stream_fuzzy_hash.c | 737 - src/inc_internal/FQDN_engine.h | 69 - src/inc_internal/Maat_ex_data.h | 30 - src/inc_internal/Maat_garbage_collection.h | 10 - src/inc_internal/Maat_hierarchy.h | 46 - src/inc_internal/Maat_limits.h | 10 - src/inc_internal/Maat_rule_internal.h | 416 - src/inc_internal/Maat_table_runtime.h | 120 - src/inc_internal/Maat_table_schema.h | 236 - src/inc_internal/Maat_utils.h | 100 - src/inc_internal/UniversalBoolMatch.h | 46 - src/inc_internal/alignment_int64.h | 57 - src/inc_internal/cJSON.h | 277 - src/inc_internal/config_monitor.h | 15 - src/inc_internal/dynamic_array.h | 13 - src/inc_internal/hyperscan_adapter.h | 42 - src/inc_internal/interval_index.h | 313 - src/inc_internal/json2iris.h | 6 - src/inc_internal/map_str2int.h | 11 - src/inc_internal/queue.h | 113 - src/inc_internal/rbtree.h | 118 - src/inc_internal/rbtree_augmented.h | 241 - src/inc_internal/sfh_internal.h | 108 - src/inc_internal/uthash/utarray.h | 247 - src/inc_internal/uthash/uthash.h | 1150 - src/inc_internal/uthash/utlist.h | 1073 - src/inc_internal/uthash/utringbuffer.h | 108 - src/inc_internal/uthash/utstack.h | 88 - src/inc_internal/uthash/utstring.h | 407 - src/inc_internal/view_only/IPMatcher.h | 117 - .../view_only/MESA_handle_logger.h | 75 - src/inc_internal/view_only/MESA_htable.h | 275 - src/inc_internal/view_only/MESA_list.h | 31 - src/inc_internal/view_only/MESA_list_queue.h | 114 - src/inc_internal/view_only/field_stat2.h | 119 - src/inc_internal/view_only/hiredis/hiredis.h | 336 - src/inc_internal/view_only/hiredis/read.h | 129 - src/inc_internal/view_only/hiredis/sds.h | 278 - src/inc_internal/view_only/hyperscan/hs.h | 51 - .../view_only/hyperscan/hs_common.h | 596 - .../view_only/hyperscan/hs_compile.h | 1224 - .../view_only/hyperscan/hs_runtime.h | 621 - src/inc_internal/view_only/rulescan.h | 322 - src/inc_internal/zt_hash.h | 234 - src/version.map | 15 - test/CMakeLists.txt | 25 - ...lder is used to set rulescan scan para.txt | 0 test/conf/config.txt | 30 - test/file_test_tableinfo.conf | 27 - test/file_test_tableinfo.conf.bak | 27 - test/json_update/corrupted.json | 32 - test/json_update/new.json | 32 - test/json_update/old.json | 33 - test/maat_demo.cpp | 1223 - test/maat_json.json | 2389 - .../full/2018-10-09/APP_COMPILE.0000050997 | 22 - .../full/2018-10-09/APP_GROUP.0000050997 | 38 - .../full/2018-10-09/APP_PAYLOAD.0000050997 | 2 - .../full/2018-10-09/APP_POLICY.0000050997 | 17 - .../DDOS_PROTECT_TARGET_IP_CB.0000050997 | 2 - .../full/2018-10-09/MM_AV_URL.0000050997 | 2 - .../full/2018-10-09/MM_COMPILE.0000050997 | 2 - .../full/2018-10-09/MM_GROUP.0000050997 | 2 - .../full/2018-10-09/NTC_ASN_IP.0000050997 | 2 - .../full/2018-10-09/NTC_BGP_AS.0000050997 | 11 - .../full/2018-10-09/NTC_COMPILE.0000050997 | 79 - .../2018-10-09/NTC_DNS_FAKE_IP_CB.0000050997 | 4 - .../full/2018-10-09/NTC_DNS_REGION.0000050997 | 16 - .../NTC_DNS_RES_STRATEGY.0000050997 | 3 - .../full/2018-10-09/NTC_FTP_URL.0000050997 | 6 - .../2018-10-09/NTC_GROUP2COMPILE.0000050997 | 105 - .../2018-10-09/NTC_HTTP_REQ_BODY.0000050997 | 5 - .../2018-10-09/NTC_HTTP_RES_BODY.0000050997 | 15 - .../full/2018-10-09/NTC_HTTP_URL.0000050997 | 10 - .../full/2018-10-09/NTC_MAIL_BODY.0000050997 | 3 - .../full/2018-10-09/NTC_MAIL_HDR.0000050997 | 15 - .../2018-10-09/NTC_UNIVERSAL_IP.0000050997 | 24 - .../NTC_UNIVERSAL_PROTO_TYPE.0000050997 | 24 - .../2018-10-09/WHITE_LIST_COMPILE.0000050997 | 2 - .../2018-10-09/WHITE_LIST_GROUP.0000050997 | 2 - .../full/2018-10-09/WHITE_LIST_IP.0000050997 | 2 - .../full/index/full_config_index.0000050997 | 26 - test/perf_test_maatframe.cpp | 1005 - test/reset_redis4maat.sh | 11 - test/rule/full/COMPILE.local | 13 - test/rule/full/CONTENT_SIZE.local | 3 - test/rule/full/FILE_DIGEST.local | 2 - test/rule/full/GROUP.local | 16 - test/rule/full/HTTP_HOST.local | 2 - test/rule/full/HTTP_REGION.local | 2 - test/rule/full/HTTP_URL.local | 6 - test/rule/full/HTTP_URL.local.encrypt | 2 - test/rule/full/IP_CONFIG.local | 3 - test/rule/full/KEYWORDS_TABLE.local | 4 - test/rule/full/QD_ENTRY_INFO.local | 4 - test/rule/full/TEST_PLUGIN_TABLE.local | 4 - test/rule/full/TEST_PLUGIN_TABLE_NO1.local | 4 - test/rule/full/TEST_PLUGIN_TABLE_NO2.local | 1 - .../full/index/full_config_index.0000000001 | 11 - test/rule/inc/COMPILE.local | 13 - test/rule/inc/CONTENT_SIZE.local | 3 - test/rule/inc/FILE_DIGEST.local | 2 - test/rule/inc/GROUP.local | 16 - test/rule/inc/HTTP_HOST.local | 2 - test/rule/inc/HTTP_REGION.local | 2 - test/rule/inc/HTTP_URL.local | 6 - test/rule/inc/IP_CONFIG.local | 3 - test/rule/inc/KEYWORDS_TABLE.local | 3 - test/rule/inc/QD_ENTRY_INFO.local | 4 - test/rule/inc/TEST_PLUGIN_TABLE.local | 4 - test/rule/inc/TEST_PLUGIN_TABLE_NO1.local | 4 - test/rule/inc/TEST_PLUGIN_TABLE_NO2.local | 1 - .../inc/index/full_config_index.0000000002 | 10 - test/table_info.conf | 65 - test/test_igraph.cpp | 62 - test/test_maatframe.cpp | 4918 - test/test_streamfiles/stream_dump.1 | 55 - test/test_streamfiles/stream_dump.2 | 25 - test/test_streamfiles/stream_dump.3 | 95 - test/test_streamfiles/stream_dump.4 | 106 - test/test_streamfiles/stream_dump.5 | 179 - test/test_streamfiles/stream_dump.6 | 114 - test/test_streamfiles/stream_dump.7 | 136 - test/test_streamfiles/stream_dump.8 | 139 - test/test_streamfiles/stream_dump.9 | 11 - test/testdata/bool-matcher-test-exprs.txt | 77893 ---------------- test/testdata/charsetWindows1251.txt | 48 - test/testdata/digest_test.data | Bin 1160164 -> 0 bytes test/testdata/jd.com.html | 968 - test/testdata/mesa_logo.jpg | Bin 107970 -> 0 bytes .../original_Uygur_webpage.html | 166 - test/testdata_uni2ascii/original_uy.txt | 2 - test/testdata_uni2ascii/qq_mail_https.txt | 327 - test/testdata_uni2ascii/sina_read_mail.txt | 1 - test/testdata_uni2ascii/sohu_mail_unicode.txt | 1 - test/tsg_tableinfo.conf | 1 - test/tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k | 1001 - .../full_config_index.00000000000000106967 | 2 - tools/CMakeLists.txt | 16 - tools/digest_gen.c | 165 - tools/maat_debug_tool.cpp | 266 - tools/maat_redis_tool.cpp | 377 - vendor/CMakeLists.txt | 62 - vendor/googletest-release-1.8.0.tar.gz | Bin 1281617 -> 0 bytes vendor/hiredis-1.0.2.tar.gz | Bin 98139 -> 0 bytes vendor/igraph-0.7.1.tar.gz | Bin 2967134 -> 0 bytes vendor/ipmatcher-v1.1.zip | Bin 87429 -> 0 bytes 191 files changed, 23 insertions(+), 127705 deletions(-) delete mode 100644 .gitignore delete mode 100644 .gitlab-ci.yml delete mode 100644 CMakeLists.txt delete mode 100644 Makefile delete mode 100644 ancient_history.txt delete mode 100644 autorevision.sh delete mode 100644 ci/get-nprocessors.sh delete mode 100644 ci/perpare_pulp3_netrc.sh delete mode 100644 ci/travis.sh delete mode 100644 cmake/Package.cmake delete mode 100644 cmake/Version.cmake delete mode 100644 cmake/changelog.sh delete mode 100644 inc/Maat_command.h delete mode 100644 inc/Maat_rule.h delete mode 100644 inc/bool_matcher.h delete mode 100644 inc/gram_index_engine.h delete mode 100644 inc/stream_fuzzy_hash.h create mode 100644 include/maat/maat.h delete mode 100644 lib/.gitignore delete mode 100644 readme.md delete mode 100644 src/CMakeLists.txt delete mode 100644 src/entry/.gitignore delete mode 100644 src/entry/FQDN_engine.cpp delete mode 100644 src/entry/Maat_api.cpp delete mode 100644 src/entry/Maat_command.cpp delete mode 100644 src/entry/Maat_ex_data.cpp delete mode 100644 src/entry/Maat_garbage_collection.cpp delete mode 100644 src/entry/Maat_hierarchy.cpp delete mode 100644 src/entry/Maat_rule.cpp delete mode 100644 src/entry/Maat_stat.cpp delete mode 100644 src/entry/Maat_table_runtime.cpp delete mode 100644 src/entry/Maat_table_schema.cpp delete mode 100644 src/entry/Maat_utils.cpp delete mode 100644 src/entry/UniversalBoolMatch.cpp delete mode 100644 src/entry/bool_matcher.cpp delete mode 100644 src/entry/cJSON.c delete mode 100644 src/entry/config_monitor.cpp delete mode 100644 src/entry/dynamic_array.cpp delete mode 100644 src/entry/gram_index_engine.c delete mode 100644 src/entry/hyperscan_adapter.cpp delete mode 100644 src/entry/interval_index.c delete mode 100644 src/entry/json2iris.cpp delete mode 100644 src/entry/map_str2int.cpp delete mode 100644 src/entry/rbtree.c delete mode 100644 src/entry/stream_fuzzy_hash.c delete mode 100644 src/inc_internal/FQDN_engine.h delete mode 100644 src/inc_internal/Maat_ex_data.h delete mode 100644 src/inc_internal/Maat_garbage_collection.h delete mode 100644 src/inc_internal/Maat_hierarchy.h delete mode 100644 src/inc_internal/Maat_limits.h delete mode 100644 src/inc_internal/Maat_rule_internal.h delete mode 100644 src/inc_internal/Maat_table_runtime.h delete mode 100644 src/inc_internal/Maat_table_schema.h delete mode 100644 src/inc_internal/Maat_utils.h delete mode 100644 src/inc_internal/UniversalBoolMatch.h delete mode 100644 src/inc_internal/alignment_int64.h delete mode 100644 src/inc_internal/cJSON.h delete mode 100644 src/inc_internal/config_monitor.h delete mode 100644 src/inc_internal/dynamic_array.h delete mode 100644 src/inc_internal/hyperscan_adapter.h delete mode 100644 src/inc_internal/interval_index.h delete mode 100644 src/inc_internal/json2iris.h delete mode 100644 src/inc_internal/map_str2int.h delete mode 100644 src/inc_internal/queue.h delete mode 100644 src/inc_internal/rbtree.h delete mode 100644 src/inc_internal/rbtree_augmented.h delete mode 100644 src/inc_internal/sfh_internal.h delete mode 100644 src/inc_internal/uthash/utarray.h delete mode 100644 src/inc_internal/uthash/uthash.h delete mode 100644 src/inc_internal/uthash/utlist.h delete mode 100644 src/inc_internal/uthash/utringbuffer.h delete mode 100644 src/inc_internal/uthash/utstack.h delete mode 100644 src/inc_internal/uthash/utstring.h delete mode 100644 src/inc_internal/view_only/IPMatcher.h delete mode 100644 src/inc_internal/view_only/MESA_handle_logger.h delete mode 100644 src/inc_internal/view_only/MESA_htable.h delete mode 100644 src/inc_internal/view_only/MESA_list.h delete mode 100644 src/inc_internal/view_only/MESA_list_queue.h delete mode 100644 src/inc_internal/view_only/field_stat2.h delete mode 100644 src/inc_internal/view_only/hiredis/hiredis.h delete mode 100644 src/inc_internal/view_only/hiredis/read.h delete mode 100644 src/inc_internal/view_only/hiredis/sds.h delete mode 100644 src/inc_internal/view_only/hyperscan/hs.h delete mode 100644 src/inc_internal/view_only/hyperscan/hs_common.h delete mode 100644 src/inc_internal/view_only/hyperscan/hs_compile.h delete mode 100644 src/inc_internal/view_only/hyperscan/hs_runtime.h delete mode 100644 src/inc_internal/view_only/rulescan.h delete mode 100644 src/inc_internal/zt_hash.h delete mode 100644 src/version.map delete mode 100644 test/CMakeLists.txt delete mode 100644 test/conf/This folder is used to set rulescan scan para.txt delete mode 100644 test/conf/config.txt delete mode 100644 test/file_test_tableinfo.conf delete mode 100644 test/file_test_tableinfo.conf.bak delete mode 100644 test/json_update/corrupted.json delete mode 100644 test/json_update/new.json delete mode 100644 test/json_update/old.json delete mode 100644 test/maat_demo.cpp delete mode 100644 test/maat_json.json delete mode 100644 test/ntcrule/full/2018-10-09/APP_COMPILE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/APP_GROUP.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/APP_PAYLOAD.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/APP_POLICY.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/DDOS_PROTECT_TARGET_IP_CB.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/MM_AV_URL.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/MM_COMPILE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/MM_GROUP.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_ASN_IP.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_BGP_AS.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_COMPILE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_DNS_FAKE_IP_CB.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_DNS_REGION.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_DNS_RES_STRATEGY.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_FTP_URL.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_GROUP2COMPILE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_HTTP_REQ_BODY.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_HTTP_RES_BODY.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_HTTP_URL.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_MAIL_BODY.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_MAIL_HDR.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_UNIVERSAL_IP.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/NTC_UNIVERSAL_PROTO_TYPE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/WHITE_LIST_COMPILE.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/WHITE_LIST_GROUP.0000050997 delete mode 100644 test/ntcrule/full/2018-10-09/WHITE_LIST_IP.0000050997 delete mode 100644 test/ntcrule/full/index/full_config_index.0000050997 delete mode 100644 test/perf_test_maatframe.cpp delete mode 100644 test/reset_redis4maat.sh delete mode 100644 test/rule/full/COMPILE.local delete mode 100644 test/rule/full/CONTENT_SIZE.local delete mode 100644 test/rule/full/FILE_DIGEST.local delete mode 100644 test/rule/full/GROUP.local delete mode 100644 test/rule/full/HTTP_HOST.local delete mode 100644 test/rule/full/HTTP_REGION.local delete mode 100644 test/rule/full/HTTP_URL.local delete mode 100644 test/rule/full/HTTP_URL.local.encrypt delete mode 100644 test/rule/full/IP_CONFIG.local delete mode 100644 test/rule/full/KEYWORDS_TABLE.local delete mode 100644 test/rule/full/QD_ENTRY_INFO.local delete mode 100644 test/rule/full/TEST_PLUGIN_TABLE.local delete mode 100644 test/rule/full/TEST_PLUGIN_TABLE_NO1.local delete mode 100644 test/rule/full/TEST_PLUGIN_TABLE_NO2.local delete mode 100644 test/rule/full/index/full_config_index.0000000001 delete mode 100644 test/rule/inc/COMPILE.local delete mode 100644 test/rule/inc/CONTENT_SIZE.local delete mode 100644 test/rule/inc/FILE_DIGEST.local delete mode 100644 test/rule/inc/GROUP.local delete mode 100644 test/rule/inc/HTTP_HOST.local delete mode 100644 test/rule/inc/HTTP_REGION.local delete mode 100644 test/rule/inc/HTTP_URL.local delete mode 100644 test/rule/inc/IP_CONFIG.local delete mode 100644 test/rule/inc/KEYWORDS_TABLE.local delete mode 100644 test/rule/inc/QD_ENTRY_INFO.local delete mode 100644 test/rule/inc/TEST_PLUGIN_TABLE.local delete mode 100644 test/rule/inc/TEST_PLUGIN_TABLE_NO1.local delete mode 100644 test/rule/inc/TEST_PLUGIN_TABLE_NO2.local delete mode 100644 test/rule/inc/index/full_config_index.0000000002 delete mode 100644 test/table_info.conf delete mode 100644 test/test_igraph.cpp delete mode 100644 test/test_maatframe.cpp delete mode 100644 test/test_streamfiles/stream_dump.1 delete mode 100644 test/test_streamfiles/stream_dump.2 delete mode 100644 test/test_streamfiles/stream_dump.3 delete mode 100644 test/test_streamfiles/stream_dump.4 delete mode 100644 test/test_streamfiles/stream_dump.5 delete mode 100644 test/test_streamfiles/stream_dump.6 delete mode 100644 test/test_streamfiles/stream_dump.7 delete mode 100644 test/test_streamfiles/stream_dump.8 delete mode 100644 test/test_streamfiles/stream_dump.9 delete mode 100644 test/testdata/bool-matcher-test-exprs.txt delete mode 100644 test/testdata/charsetWindows1251.txt delete mode 100644 test/testdata/digest_test.data delete mode 100644 test/testdata/jd.com.html delete mode 100644 test/testdata/mesa_logo.jpg delete mode 100644 test/testdata_uni2ascii/original_Uygur_webpage.html delete mode 100644 test/testdata_uni2ascii/original_uy.txt delete mode 100644 test/testdata_uni2ascii/qq_mail_https.txt delete mode 100644 test/testdata_uni2ascii/sina_read_mail.txt delete mode 100644 test/testdata_uni2ascii/sohu_mail_unicode.txt delete mode 100644 test/tsg_tableinfo.conf delete mode 100644 test/tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k delete mode 100644 test/tsgrule/full/index/full_config_index.00000000000000106967 delete mode 100644 tools/CMakeLists.txt delete mode 100644 tools/digest_gen.c delete mode 100644 tools/maat_debug_tool.cpp delete mode 100644 tools/maat_redis_tool.cpp delete mode 100644 vendor/CMakeLists.txt delete mode 100644 vendor/googletest-release-1.8.0.tar.gz delete mode 100644 vendor/hiredis-1.0.2.tar.gz delete mode 100644 vendor/igraph-0.7.1.tar.gz delete mode 100644 vendor/ipmatcher-v1.1.zip diff --git a/.gitignore b/.gitignore deleted file mode 100644 index f0519b3..0000000 --- a/.gitignore +++ /dev/null @@ -1,45 +0,0 @@ -# Created by https://www.gitignore.io/api/c++,clion - -### C++ ### -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# CMake -cmake-build-*/ -build/* - -# Clion -.idea/ - -# Vscode -.vscode/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 1c2b299..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,224 +0,0 @@ -variables: - GIT_STRATEGY: "clone" - BUILD_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master" - BUILD_IMAGE_CENTOS8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux" - BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ - INSTALL_DEPENDENCY_LIBRARY: libMESA_handle_logger-devel libMESA_htable-devel pcre-devel librulescan-devel libMESA_field_stat2-devel libMESA_field_stat-devel sapp-devel framework_env openssl-devel libasan - -stages: -- build - -.build_before_script: - before_script: - - mkdir -p $BUILD_PADDING_PREFIX/$CI_PROJECT_NAMESPACE/ - - ln -s $CI_PROJECT_DIR $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH - - cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH - - chmod +x ./ci/travis.sh - - yum makecache - - yum install -y elfutils-libelf-devel - - -.build_by_travis_for_centos7: - stage: build - image: $BUILD_IMAGE_CENTOS7 - extends: .build_before_script - script: - - yum install -y libmnl-devel - - yum install -y libnfnetlink-devel - - ./ci/travis.sh - - cd build - tags: - - share - -.build_by_travis_for_centos8: - stage: build - image: $BUILD_IMAGE_CENTOS8 - extends: .build_before_script - script: - - dnf --enablerepo=powertools install -y libmnl-devel - - dnf --enablerepo=powertools install -y libnfnetlink-devel - - ./ci/travis.sh - tags: - - share - -branch_build_debug_for_centos7: - stage: build - extends: .build_by_travis_for_centos7 - variables: - BUILD_TYPE: Debug - except: - - /^develop.*$/i - - /^master.*$/i - - tags - -branch_build_release_for_centos7: - stage: build - variables: - BUILD_TYPE: RelWithDebInfo - extends: .build_by_travis_for_centos7 - except: - - /^develop.*$/i - - /^master.*$/i - - tags - -develop_build_debug_for_centos7: - stage: build - extends: .build_by_travis_for_centos7 - variables: - BUILD_TYPE: Debug - PACKAGE: 1 - UPLOAD: 1 - ASAN_OPTION: ADDRESS - TESTING_VERSION_BUILD: 1 - PULP3_REPO_NAME: framework-testing-x86_64.el7 - PULP3_DIST_NAME: framework-testing-x86_64.el7 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - /^develop.*$/i - - /^master.*$/i - -develop_build_release_for_centos7: - stage: build - extends: .build_by_travis_for_centos7 - variables: - BUILD_TYPE: RelWithDebInfo - PACKAGE: 1 - UPLOAD: 1 - TESTING_VERSION_BUILD: 1 - PULP3_REPO_NAME: framework-testing-x86_64.el7 - PULP3_DIST_NAME: framework-testing-x86_64.el7 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - /^develop.*$/i - - /^master.*$/i - -release_build_debug_for_centos7: - stage: build - variables: - BUILD_TYPE: Debug - PACKAGE: 1 - UPLOAD: 1 - PULP3_REPO_NAME: framework-stable-x86_64.el7 - PULP3_DIST_NAME: framework-stable-x86_64.el7 - extends: .build_by_travis_for_centos7 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - tags - -release_build_release_for_centos7: - stage: build - variables: - BUILD_TYPE: RelWithDebInfo - PACKAGE: 1 - UPLOAD: 1 - UPLOAD_SYMBOL_FILES: 1 - SYMBOL_TARGET: libmaatframe - PULP3_REPO_NAME: framework-stable-x86_64.el7 - PULP3_DIST_NAME: framework-stable-x86_64.el7 - extends: .build_by_travis_for_centos7 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - tags - -branch_build_debug_for_centos8: - stage: build - extends: .build_by_travis_for_centos8 - variables: - BUILD_TYPE: Debug - except: - - /^develop.*$/i - - /^master.*$/i - - tags - -branch_build_release_for_centos8: - stage: build - variables: - BUILD_TYPE: RelWithDebInfo - extends: .build_by_travis_for_centos8 - except: - - /^develop.*$/i - - /^master.*$/i - - tags - -develop_build_debug_for_centos8: - stage: build - extends: .build_by_travis_for_centos8 - variables: - BUILD_TYPE: Debug - PACKAGE: 1 - UPLOAD: 1 - ASAN_OPTION: ADDRESS - TESTING_VERSION_BUILD: 1 - PULP3_REPO_NAME: framework-testing-x86_64.el8 - PULP3_DIST_NAME: framework-testing-x86_64.el8 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - /^develop.*$/i - - /^master.*$/i - -develop_build_release_for_centos8: - stage: build - extends: .build_by_travis_for_centos8 - variables: - BUILD_TYPE: RelWithDebInfo - PACKAGE: 1 - UPLOAD: 1 - TESTING_VERSION_BUILD: 1 - PULP3_REPO_NAME: framework-testing-x86_64.el8 - PULP3_DIST_NAME: framework-testing-x86_64.el8 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - /^develop.*$/i - - /^master.*$/i - -release_build_debug_for_centos8: - stage: build - variables: - BUILD_TYPE: Debug - PACKAGE: 1 - UPLOAD: 1 - PULP3_REPO_NAME: framework-stable-x86_64.el8 - PULP3_DIST_NAME: framework-stable-x86_64.el8 - extends: .build_by_travis_for_centos8 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-debug" - paths: - - build/*.rpm - only: - - tags - -release_build_release_for_centos8: - stage: build - variables: - BUILD_TYPE: RelWithDebInfo - PACKAGE: 1 - UPLOAD: 1 - UPLOAD_SYMBOL_FILES: 1 - SYMBOL_TARGET: libmaatframe - PULP3_REPO_NAME: framework-stable-x86_64.el8 - PULP3_DIST_NAME: framework-stable-x86_64.el8 - extends: .build_by_travis_for_centos8 - artifacts: - name: "maatframe-$CI_COMMIT_REF_NAME-release" - paths: - - build/*.rpm - only: - - tags diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 28e9268..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -cmake_minimum_required (VERSION 2.8) -set(lib_name maatframe) -project (maatframe) - -set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) -include(Version) - -SET(CMAKE_INSTALL_PREFIX /opt/MESA/) - -set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall) -set(MAAT_DEPEND_DYN_LIB MESA_handle_logger MESA_htable pcre rulescan pthread m pcre MESA_field_stat2 crypto z) -include_directories(${PROJECT_SOURCE_DIR}/inc/) -include_directories(/opt/MESA/include/) - -#for ASAN -set(ASAN_OPTION "OFF" CACHE STRING " set asan type chosen by the user, using OFF as default") -set_property(CACHE ASAN_OPTION PROPERTY STRINGS OFF ADDRESS THREAD) -message(STATUS "ASAN_OPTION='${ASAN_OPTION}'") - -if(ASAN_OPTION MATCHES "ADDRESS") - set(CMAKE_C_FLAGS "${CMAKADDRESS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") -elseif(ASAN_OPTION MATCHES "THREAD") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") -endif() -# end of for ASAN - -enable_testing() - -add_subdirectory (vendor) -add_subdirectory (src) -add_subdirectory (test) -add_subdirectory (tools) - -include(Package) diff --git a/Makefile b/Makefile deleted file mode 100644 index ac23531..0000000 --- a/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -BUILD_DIR = $(CURDIR)/build -LOCAL_DIR = $(CURDIR) -DEBUG_FLAGS = -DCMAKE_BUILD_TYPE=Debug -REL_FLAGS = -DCMAKE_BUILD_TYPE=RelWithDebInfo - -ifneq ($(INSTALL_PREFIX),) -DEBUG_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -REL_FLAGS += -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -endif - -all: _make_build_dir _compile_rel - -PHONY: all _make_build_dir _compile_debug _compile_rel _install \ - build_release build_debug install - -_make_build_dir: - mkdir -p $(BUILD_DIR) - -_compile_debug: - cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(DEBUG_FLAGS) && make - -_compile_rel: - cd $(BUILD_DIR) && cmake $(LOCAL_DIR) $(REL_FLAGS) && make - -_install: - cd $(BUILD_DIR) && make install -_package: - cd $(BUILD_DIR) && make package -_clean: - rm -rf $(BUILD_DIR) - -# Release Version, No Debug Symbol and Optimized with -O2 -release: _make_build_dir _compile_rel -# Debug Version, Optimized with -O0 -debug: _make_build_dir _compile_debug -# Install -install: _install -# Package -package: _package -# Clean -clean: _clean diff --git a/ancient_history.txt b/ancient_history.txt deleted file mode 100644 index 6076da0..0000000 --- a/ancient_history.txt +++ /dev/null @@ -1,48 +0,0 @@ -Author:Zheng Chao zhengchao@iie.ac.cn -2014-04-01 create this project; -2014-08-19 first online version; -2014-09-09 support expr offset; -2014-09-26 add group function; -2014-12-03 support regex grouping; -2014-12-17 write some comments in Maat_rule.h; -2014-12-22 add group_num in compile config table; -2015-01-04 make string match not case sensitive,fix garbage_bagging malloc wrong size bug; -2015-01-07 replace assert of database rule format,unescape '&' of EXPR_TYPE_OFFSET; -2015-01-22 ouput IRIS index file format error;verify string scan input data -and data len; -2015-01-28 enfore parameter check in Maat_stream_scan_string_detail to -optimize performance; -2015-02-04 support stream scan data with offset; -2015-02-20 add JSON config mode and add a demo; -2015-03-02 when use iconv_convert from gbk to big5,automaticly alternate gbk to -gb2312; -2015-04-13 1)Maat_xx_scan will return 0 if table has no config instead of -1; - 2)use my_scandir as replacement of glibc's scandir to adapt dictator - malloc wrap; - 3)if iconv take no effect,jump over this string; - 4)compile shortcut for performance; - 5)verify if region id is unique; -2015-04-20 fix Maat_stream_scan_string_detail and Maat_stream_scan_string wrong - data len when open cross caching; -2015-04-21 fix add_group_to_compile bug on compatible to none-group mode; -2015-04-29 fix false hit compile rule of one more region bug; -2015-05-06 fix add_group_to_compile return add failed status bug; -2015-05-07 1)use rwlock instead of mutex in _mi_rule; - 2)fix inc postpone update bug; -2015-05-21 1)addapt rulescan ip addr host order requirement; - 2) use readdir_r in my_scandir; - 3) fix segmentfault when have no config,again; - 4) maat_json support plugin table; -2015-07-03 1)print error when Maat_summon_feather with a dir has no valid -indexfile; - 2)check AND_EXPR's match method; - 3)remove restriction of IP table's protocol; - 4)fix invalid write in insert_set_id function; -2015-07-06 1)handle wrong expr format like "aa&&bb" and "aa&bb&"; - 2)iconv_convert performance optimized; -2015-10-19 check table_type in callback register; -2015-11-09 1)add digest feature; - 2) split some code from Maat_rule.cpp to Maat_api.cpp; -2015-12-24 change plugin table update mechanism to save memory; -2016-01-20 maat_finish_cb adapt empty inc callback on a NULL scanner; -2016-01-31 trigger plugin table's callback ONLY on its table's changed; diff --git a/autorevision.sh b/autorevision.sh deleted file mode 100644 index 3baa179..0000000 --- a/autorevision.sh +++ /dev/null @@ -1,1268 +0,0 @@ -#!/bin/sh - -# Copyright (c) 2012 - 2016 dak180 and contributors. See -# https://opensource.org/licenses/mit-license.php or the included -# COPYING.md for licence terms. -# -# autorevision - extracts metadata about the head version from your -# repository. - -# Usage message. -arUsage() { - cat > "/dev/stderr" << EOF -usage: autorevision {-t output-type | -s symbol} [-o cache-file [-f] ] [-V] - Options include: - -t output-type = specify output type - -s symbol = specify symbol output - -o cache-file = specify cache file location - -f = force the use of cache data - -U = check for untracked files in svn - -V = emit version and exit - -? = help message - -The following are valid output types: - clojure = clojure file - c = C/C++ file - h = Header for use with c/c++ - hpp = Alternate C++ header strings with namespace - ini = INI file - java = Java file - javaprop = Java properties file - js = javascript file - json = JSON file - lua = Lua file - m4 = m4 file - matlab = matlab file - octave = octave file - php = PHP file - pl = Perl file - py = Python file - rpm = rpm file - scheme = scheme file - sh = Bash sytax - swift = Swift file - tex = (La)TeX file - xcode = Header useful for populating info.plist files - cmake = CMake file - - -The following are valid symbols: - VCS_TYPE - VCS_BASENAME - VCS_UUID - VCS_NUM - VCS_DATE - VCS_BRANCH - VCS_TAG - VCS_TICK - VCS_EXTRA - VCS_FULL_HASH - VCS_SHORT_HASH - VCS_WC_MODIFIED - VCS_ACTION_STAMP -EOF - exit 1 -} - -# Config -ARVERSION="&&ARVERSION&&" -TARGETFILE="/dev/stdout" -while getopts ":t:o:s:VfU" OPTION; do - case "${OPTION}" in - t) - AFILETYPE="${OPTARG}" - ;; - o) - CACHEFILE="${OPTARG}" - ;; - f) - CACHEFORCE="1" - ;; - s) - VAROUT="${OPTARG}" - ;; - U) - UNTRACKEDFILES="1" - ;; - V) - echo "autorevision ${ARVERSION}" - exit 0 - ;; - ?) - # If an unknown flag is used (or -?): - arUsage - ;; - esac -done - -if [ ! -z "${VAROUT}" ] && [ ! -z "${AFILETYPE}" ]; then - # If both -s and -t are specified: - echo "error: Improper argument combination." 1>&2 - exit 1 -elif [ -z "${VAROUT}" ] && [ -z "${AFILETYPE}" ]; then - # If neither -s or -t are specified: - arUsage -elif [ -z "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then - # If -f is specified without -o: - arUsage -elif [ ! -f "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then - # If we are forced to use the cache but it does not exist. - echo "error: Cache forced but no cache found." 1>&2 - exit 1 -fi - -# Make sure that the path we are given is one we can source -# (dash, we are looking at you). -if [ ! -z "${CACHEFILE}" ] && ! echo "${CACHEFILE}" | grep -q '^\.*/'; then - CACHEFILE="./${CACHEFILE}" -fi - -GENERATED_HEADER="Generated by autorevision - do not hand-hack!" - -# Functions to extract data from different repo types. -# For git repos -# shellcheck disable=SC2039,SC2164,SC2155 -gitRepo() { - local oldPath="${PWD}" - - cd "$(git rev-parse --show-toplevel)" - - VCS_TYPE="git" - - VCS_BASENAME="$(basename "${PWD}")" - - VCS_UUID="$(git rev-list --max-parents=0 --date-order --reverse HEAD 2>/dev/null | sed -n 1p)" - if [ -z "${VCS_UUID}" ]; then - VCS_UUID="$(git rev-list --topo-order HEAD | tail -n 1)" - fi - - # Is the working copy clean? - test -z "$(git status --untracked-files=normal --porcelain)" - VCS_WC_MODIFIED="${?}" - - # Enumeration of changesets - VCS_NUM="$(git rev-list --count HEAD 2>/dev/null)" - if [ -z "${VCS_NUM}" ]; then - echo "warning: Counting the number of revisions may be slower due to an outdated git version less than 1.7.2.3. If something breaks, please update it." 1>&2 - VCS_NUM="$(git rev-list HEAD | wc -l)" - fi - - # This may be a git-svn remote. If so, report the Subversion revision. - if [ -z "$(git config svn-remote.svn.url 2>/dev/null)" ]; then - # The full revision hash - VCS_FULL_HASH="$(git rev-parse HEAD)" - - # The short hash - VCS_SHORT_HASH="$(echo "${VCS_FULL_HASH}" | cut -b 1-7)" - else - # The git-svn revision number - VCS_FULL_HASH="$(git svn find-rev HEAD)" - VCS_SHORT_HASH="${VCS_FULL_HASH}" - fi - - # Current branch - VCS_BRANCH="$(git rev-parse --symbolic-full-name --verify "$(git name-rev --name-only --no-undefined HEAD 2>/dev/null)" 2>/dev/null | sed -e 's:refs/heads/::' | sed -e 's:refs/::')" - - # Cache the description - local DESCRIPTION="$(git describe --long --tags 2>/dev/null)" - - # Current or last tag ancestor (empty if no tags) - VCS_TAG="$(echo "${DESCRIPTION}" | sed -e "s:-g${VCS_SHORT_HASH}\$::" -e 's:-[0-9]*$::')" - - # Distance to last tag or an alias of VCS_NUM if there is no tag - if [ ! -z "${DESCRIPTION}" ]; then - VCS_TICK="$(echo "${DESCRIPTION}" | sed -e "s:${VCS_TAG}-::" -e "s:-g${VCS_SHORT_HASH}::")" - else - VCS_TICK="${VCS_NUM}" - fi - - # Date of the current commit - VCS_DATE="$(TZ=UTC git show -s --date=iso-strict-local --pretty=format:%ad | sed -e 's|+00:00|Z|')" - if [ -z "${VCS_DATE}" ]; then - echo "warning: Action stamps require git version 2.7+." 1>&2 - VCS_DATE="$(git log -1 --pretty=format:%ci | sed -e 's: :T:' -e 's: ::' -e 's|+00:00|Z|')" - local ASdis="1" - fi - - # Action Stamp - if [ -z "${ASdis}" ]; then - VCS_ACTION_STAMP="${VCS_DATE}!$(git show -s --pretty=format:%cE)" - else - VCS_ACTION_STAMP="" - fi - - cd "${oldPath}" -} - -# For hg repos -# shellcheck disable=SC2039,SC2164 -hgRepo() { - local oldPath="${PWD}" - - cd "$(hg root)" - - VCS_TYPE="hg" - - VCS_BASENAME="$(basename "${PWD}")" - - VCS_UUID="$(hg log -r "0" -l 1 --template '{node}\n')" - - # Is the working copy clean? - test -z "$(hg status -duram)" - VCS_WC_MODIFIED="${?}" - - # Enumeration of changesets - VCS_NUM="$(hg id -n | tr -d '+')" - - # The full revision hash - VCS_FULL_HASH="$(hg log -r "${VCS_NUM}" -l 1 --template '{node}\n')" - - # The short hash - VCS_SHORT_HASH="$(hg id -i | tr -d '+')" - - # Current bookmark (bookmarks are roughly equivalent to git's branches) - # or branch if no bookmark - VCS_BRANCH="$(hg id -B | cut -d ' ' -f 1)" - # Fall back to the branch if there are no bookmarks - if [ -z "${VCS_BRANCH}" ]; then - VCS_BRANCH="$(hg id -b)" - fi - - # Current or last tag ancestor (excluding auto tags, empty if no tags) - VCS_TAG="$(hg log -r "${VCS_NUM}" -l 1 --template '{latesttag}\n' 2>/dev/null | sed -e 's:qtip::' -e 's:tip::' -e 's:qbase::' -e 's:qparent::' -e "s:$(hg --config 'extensions.color=' --config 'extensions.mq=' --color never qtop 2>/dev/null)::" | cut -d ' ' -f 1)" - - # Distance to last tag or an alias of VCS_NUM if there is no tag - if [ ! -z "${VCS_TAG}" ]; then - VCS_TICK="$(hg log -r "${VCS_NUM}" -l 1 --template '{latesttagdistance}\n' 2>/dev/null)" - else - VCS_TICK="${VCS_NUM}" - fi - - # Date of the current commit - VCS_DATE="$(hg log -r "${VCS_NUM}" -l 1 --template '{date|isodatesec}\n' 2>/dev/null | sed -e 's: :T:' -e 's: ::' -e 's|+00:00|Z|')" - - # Action Stamp - VCS_ACTION_STAMP="$(TZ=UTC hg log -r "${VCS_NUM}" -l 1 --template '{date|localdate|rfc3339date}\n' 2>/dev/null | sed -e 's|+00:00|Z|')!$(hg log -r "${VCS_NUM}" -l 1 --template '{author|email}\n' 2>/dev/null)" - - cd "${oldPath}" -} - -# For bzr repos -# shellcheck disable=SC2039,SC2164 -bzrRepo() { - local oldPath="${PWD}" - - cd "$(bzr root)" - - VCS_TYPE="bzr" - - VCS_BASENAME="$(basename "${PWD}")" - - # Currently unimplemented because more investigation is needed. - VCS_UUID="" - - # Is the working copy clean? - bzr version-info --custom --template='{clean}\n' | grep -q '1' - VCS_WC_MODIFIED="${?}" - - # Enumeration of changesets - VCS_NUM="$(bzr revno)" - - # The full revision hash - VCS_FULL_HASH="$(bzr version-info --custom --template='{revision_id}\n')" - - # The short hash - VCS_SHORT_HASH="${VCS_NUM}" - - # Nick of the current branch - VCS_BRANCH="$(bzr nick)" - - # Current or last tag ancestor (excluding auto tags, empty if no tags) - VCS_TAG="$(bzr tags --sort=time | sed '/?$/d' | tail -n1 | cut -d ' ' -f1)" - - # Distance to last tag or an alias of VCS_NUM if there is no tag - if [ ! -z "${VCS_TAG}" ]; then - VCS_TICK="$(bzr log --line -r "tag:${VCS_TAG}.." | tail -n +2 | wc -l | sed -e 's:^ *::')" - else - VCS_TICK="${VCS_NUM}" - fi - - # Date of the current commit - VCS_DATE="$(bzr version-info --custom --template='{date}\n' | sed -e 's: :T:' -e 's: ::')" - - # Action Stamp - # Currently unimplemented because more investigation is needed. - VCS_ACTION_STAMP="" - - cd "${oldPath}" -} - -# For svn repos -# shellcheck disable=SC2039,SC2164,SC2155 -svnRepo() { - local oldPath="${PWD}" - - VCS_TYPE="svn" - - case "${PWD}" in - /*trunk*|/*branches*|/*tags*) - local fn="${PWD}" - while [ "$(basename "${fn}")" != 'trunk' ] && [ "$(basename "${fn}")" != 'branches' ] && [ "$(basename "${fn}")" != 'tags' ] && [ "$(basename "${fn}")" != '/' ]; do - local fn="$(dirname "${fn}")" - done - local fn="$(dirname "${fn}")" - if [ "${fn}" = '/' ]; then - VCS_BASENAME="$(basename "${PWD}")" - else - VCS_BASENAME="$(basename "${fn}")" - fi - ;; - *) VCS_BASENAME="$(basename "${PWD}")" ;; - esac - - VCS_UUID="$(svn info --xml | sed -n -e 's:::' -e 's:::p')" - - # Cache svnversion output - local SVNVERSION="$(svnversion)" - - # Is the working copy clean? - echo "${SVNVERSION}" | grep -q "M" - case "${?}" in - 0) - VCS_WC_MODIFIED="1" - ;; - 1) - if [ ! -z "${UNTRACKEDFILES}" ]; then - # `svnversion` does not detect untracked files and `svn status` is really slow, so only run it if we really have to. - if [ -z "$(svn status)" ]; then - VCS_WC_MODIFIED="0" - else - VCS_WC_MODIFIED="1" - fi - else - VCS_WC_MODIFIED="0" - fi - ;; - esac - - # Enumeration of changesets - VCS_NUM="$(echo "${SVNVERSION}" | cut -d : -f 1 | sed -e 's:M::' -e 's:S::' -e 's:P::')" - - # The full revision hash - VCS_FULL_HASH="${SVNVERSION}" - - # The short hash - VCS_SHORT_HASH="${VCS_NUM}" - - # Current branch - case "${PWD}" in - /*trunk*|/*branches*|/*tags*) - local lastbase="" - local fn="${PWD}" - while : - do - base="$(basename "${fn}")" - if [ "${base}" = 'trunk' ]; then - VCS_BRANCH='trunk' - break - elif [ "${base}" = 'branches' ] || [ "${base}" = 'tags' ]; then - VCS_BRANCH="${lastbase}" - break - elif [ "${base}" = '/' ]; then - VCS_BRANCH="" - break - fi - local lastbase="${base}" - local fn="$(dirname "${fn}")" - done - ;; - *) VCS_BRANCH="" ;; - esac - - # Current or last tag ancestor (empty if no tags). But "current - # tag" can't be extracted reliably because Subversion doesn't - # have tags the way other VCSes do. - VCS_TAG="" - VCS_TICK="" - - # Date of the current commit - VCS_DATE="$(svn info --xml | sed -n -e 's:::' -e 's:::p')" - - # Action Stamp - VCS_ACTION_STAMP="${VCS_DATE}!$(svn log --xml -l 1 -r "${VCS_SHORT_HASH}" | sed -n -e 's:::' -e 's:::p')" - - cd "${oldPath}" -} - - -# Functions to output data in different formats. -# For bash output -shOutput() { - cat > "${TARGETFILE}" << EOF -# ${GENERATED_HEADER} - -VCS_TYPE="${VCS_TYPE}" -VCS_BASENAME="${VCS_BASENAME}" -VCS_UUID="${VCS_UUID}" -VCS_NUM="${VCS_NUM}" -VCS_DATE="${VCS_DATE}" -VCS_BRANCH="${VCS_BRANCH}" -VCS_TAG="${VCS_TAG}" -VCS_TICK="${VCS_TICK}" -VCS_EXTRA="${VCS_EXTRA}" - -VCS_ACTION_STAMP="${VCS_ACTION_STAMP}" -VCS_FULL_HASH="${VCS_FULL_HASH}" -VCS_SHORT_HASH="${VCS_SHORT_HASH}" - -VCS_WC_MODIFIED="${VCS_WC_MODIFIED}" - -# end -EOF -} - -# For source C output -cOutput() { - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ - -const char *VCS_TYPE = "${VCS_TYPE}"; -const char *VCS_BASENAME = "${VCS_BASENAME}"; -const char *VCS_UUID = "${VCS_UUID}"; -const int VCS_NUM = ${VCS_NUM}; -const char *VCS_DATE = "${VCS_DATE}"; -const char *VCS_BRANCH = "${VCS_BRANCH}"; -const char *VCS_TAG = "${VCS_TAG}"; -const int VCS_TICK = ${VCS_TICK}; -const char *VCS_EXTRA = "${VCS_EXTRA}"; - -const char *VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"; -const char *VCS_FULL_HASH = "${VCS_FULL_HASH}"; -const char *VCS_SHORT_HASH = "${VCS_SHORT_HASH}"; - -const int VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; - -/* end */ -EOF -} - -# For header output -hOutput() { - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ -#ifndef AUTOREVISION_H -#define AUTOREVISION_H - -#define VCS_TYPE "${VCS_TYPE}" -#define VCS_BASENAME "${VCS_BASENAME}" -#define VCS_UUID "${VCS_UUID}" -#define VCS_NUM ${VCS_NUM} -#define VCS_DATE "${VCS_DATE}" -#define VCS_BRANCH "${VCS_BRANCH}" -#define VCS_TAG "${VCS_TAG}" -#define VCS_TICK ${VCS_TICK} -#define VCS_EXTRA "${VCS_EXTRA}" - -#define VCS_ACTION_STAMP "${VCS_ACTION_STAMP}" -#define VCS_FULL_HASH "${VCS_FULL_HASH}" -#define VCS_SHORT_HASH "${VCS_SHORT_HASH}" - -#define VCS_WC_MODIFIED ${VCS_WC_MODIFIED} - -#endif - -/* end */ -EOF -} - -# A header output for use with xcode to populate info.plist strings -xcodeOutput() { - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ -#ifndef AUTOREVISION_H -#define AUTOREVISION_H - -#define VCS_TYPE ${VCS_TYPE} -#define VCS_BASENAME ${VCS_BASENAME} -#define VCS_UUID ${VCS_UUID} -#define VCS_NUM ${VCS_NUM} -#define VCS_DATE ${VCS_DATE} -#define VCS_BRANCH ${VCS_BRANCH} -#define VCS_TAG ${VCS_TAG} -#define VCS_TICK ${VCS_TICK} -#define VCS_EXTRA ${VCS_EXTRA} - -#define VCS_ACTION_STAMP ${VCS_ACTION_STAMP} -#define VCS_FULL_HASH ${VCS_FULL_HASH} -#define VCS_SHORT_HASH ${VCS_SHORT_HASH} - -#define VCS_WC_MODIFIED ${VCS_WC_MODIFIED} - -#endif - -/* end */ -EOF -} - -# For Swift output -swiftOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - # For values that may not exist depending on the type of repo we - # have read from, set them to `nil` when they are empty. - if [ -z "${VCS_UUID}" ]; then - VCS_UUID="nil" - else - VCS_UUID="\"${VCS_UUID}\"" - fi - if [ -z "${VCS_TAG}" ]; then - VCS_TAG="nil" - else - VCS_TAG="\"${VCS_TAG}\"" - fi - : "${VCS_TICK:="nil"}" - if [ -z "${VCS_EXTRA}" ]; then - VCS_EXTRA="nil" - else - VCS_EXTRA="\"${VCS_EXTRA}\"" - fi - if [ -z "${VCS_ACTION_STAMP}" ]; then - VCS_ACTION_STAMP="nil" - else - VCS_ACTION_STAMP="\"${VCS_ACTION_STAMP}\"" - fi - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ - -let VCS_TYPE = "${VCS_TYPE}" -let VCS_BASENAME = "${VCS_BASENAME}" -let VCS_UUID: String? = ${VCS_UUID} -let VCS_NUM: Int = ${VCS_NUM} -let VCS_DATE = "${VCS_DATE}" -let VCS_BRANCH: String = "${VCS_BRANCH}" -let VCS_TAG: String? = ${VCS_TAG} -let VCS_TICK: Int? = ${VCS_TICK} -let VCS_EXTRA: String? = ${VCS_EXTRA} - -let VCS_ACTION_STAMP: String? = ${VCS_ACTION_STAMP} -let VCS_FULL_HASH: String = "${VCS_FULL_HASH}" -let VCS_SHORT_HASH: String = "${VCS_SHORT_HASH}" - -let VCS_WC_MODIFIED: Bool = ${VCS_WC_MODIFIED} - -/* end */ -EOF -} - -# For Python output -pyOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="False" ;; - 1) VCS_WC_MODIFIED="True" ;; - esac - cat > "${TARGETFILE}" << EOF -# ${GENERATED_HEADER} - -VCS_TYPE = "${VCS_TYPE}" -VCS_BASENAME = "${VCS_BASENAME}" -VCS_UUID = "${VCS_UUID}" -VCS_NUM = ${VCS_NUM} -VCS_DATE = "${VCS_DATE}" -VCS_BRANCH = "${VCS_BRANCH}" -VCS_TAG = "${VCS_TAG}" -VCS_TICK = ${VCS_TICK} -VCS_EXTRA = "${VCS_EXTRA}" - -VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}" -VCS_FULL_HASH = "${VCS_FULL_HASH}" -VCS_SHORT_HASH = "${VCS_SHORT_HASH}" - -VCS_WC_MODIFIED = ${VCS_WC_MODIFIED} - -# end -EOF -} - -# For Perl output -plOutput() { - cat << EOF -# ${GENERATED_HEADER} - -\$VCS_TYPE = '${VCS_TYPE}'; -\$VCS_BASENAME = '${VCS_BASENAME}'; -\$VCS_UUID = '${VCS_UUID}'; -\$VCS_NUM = ${VCS_NUM}; -\$VCS_DATE = '${VCS_DATE}'; -\$VCS_BRANCH = '${VCS_BRANCH}'; -\$VCS_TAG = '${VCS_TAG}'; -\$VCS_TICK = ${VCS_TICK}; -\$VCS_EXTRA = '${VCS_EXTRA}'; - -\$VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}'; -\$VCS_FULL_HASH = '${VCS_FULL_HASH}'; -\$VCS_SHORT_HASH = '${VCS_SHORT_HASH}'; - -\$VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; - -# end -1; -EOF -} - -# For lua output -luaOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - cat > "${TARGETFILE}" << EOF --- ${GENERATED_HEADER} - -VCS_TYPE = "${VCS_TYPE}" -VCS_BASENAME = "${VCS_BASENAME}" -VCS_UUID = "${VCS_UUID}" -VCS_NUM = ${VCS_NUM} -VCS_DATE = "${VCS_DATE}" -VCS_BRANCH = "${VCS_BRANCH}" -VCS_TAG = "${VCS_TAG}" -VCS_TICK = ${VCS_TICK} -VCS_EXTRA = "${VCS_EXTRA}" - -VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}" -VCS_FULL_HASH = "${VCS_FULL_HASH}" -VCS_SHORT_HASH = "${VCS_SHORT_HASH}" - -VCS_WC_MODIFIED = ${VCS_WC_MODIFIED} - --- end -EOF -} - -# For php output -phpOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - cat > "${TARGETFILE}" << EOF - "${VCS_TYPE}", - "VCS_BASENAME" => "${VCS_BASENAME}", - "VCS_UUID" => "${VCS_UUID}", - "VCS_NUM" => ${VCS_NUM}, - "VCS_DATE" => "${VCS_DATE}", - "VCS_BRANCH" => "${VCS_BRANCH}", - "VCS_TAG" => "${VCS_TAG}", - "VCS_TICK" => ${VCS_TICK}, - "VCS_EXTRA" => "${VCS_EXTRA}", - "VCS_ACTION_STAMP" => "${VCS_ACTION_STAMP}", - "VCS_FULL_HASH" => "${VCS_FULL_HASH}", - "VCS_SHORT_HASH" => "${VCS_SHORT_HASH}", - "VCS_WC_MODIFIED" => ${VCS_WC_MODIFIED} -); - -# end -?> -EOF -} - -# For ini output -iniOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - cat > "${TARGETFILE}" << EOF -; ${GENERATED_HEADER} -[VCS] -VCS_TYPE = "${VCS_TYPE}" -VCS_BASENAME = "${VCS_BASENAME}" -VCS_UUID = "${VCS_UUID}" -VCS_NUM = ${VCS_NUM} -VCS_DATE = "${VCS_DATE}" -VCS_BRANCH = "${VCS_BRANCH}" -VCS_TAG = "${VCS_TAG}" -VCS_TICK = ${VCS_TICK} -VCS_EXTRA = "${VCS_EXTRA}" -VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}" -VCS_FULL_HASH = "${VCS_FULL_HASH}" -VCS_SHORT_HASH = "${VCS_SHORT_HASH}" -VCS_WC_MODIFIED = ${VCS_WC_MODIFIED} -; end -EOF -} - -# For javascript output -jsOutput() { - case "${VCS_WC_MODIFIED}" in - 1) VCS_WC_MODIFIED="true" ;; - 0) VCS_WC_MODIFIED="false" ;; - esac - cat > "${TARGETFILE}" << EOF -/** ${GENERATED_HEADER} */ - -var autorevision = { - VCS_TYPE: "${VCS_TYPE}", - VCS_BASENAME: "${VCS_BASENAME}", - VCS_UUID: "${VCS_UUID}", - VCS_NUM: ${VCS_NUM}, - VCS_DATE: "${VCS_DATE}", - VCS_BRANCH: "${VCS_BRANCH}", - VCS_TAG: "${VCS_TAG}", - VCS_TICK: ${VCS_TICK}, - VCS_EXTRA: "${VCS_EXTRA}", - - VCS_ACTION_STAMP: "${VCS_ACTION_STAMP}", - VCS_FULL_HASH: "${VCS_FULL_HASH}", - VCS_SHORT_HASH: "${VCS_SHORT_HASH}", - - VCS_WC_MODIFIED: ${VCS_WC_MODIFIED} -}; - -/** Node.js compatibility */ -if (typeof module !== 'undefined') { - module.exports = autorevision; -} - -/** end */ -EOF -} - -# For JSON output -jsonOutput() { - case "${VCS_WC_MODIFIED}" in - 1) VCS_WC_MODIFIED="true" ;; - 0) VCS_WC_MODIFIED="false" ;; - esac - cat > "${TARGETFILE}" << EOF -{ - "_comment": "${GENERATED_HEADER}", - "VCS_TYPE": "${VCS_TYPE}", - "VCS_BASENAME": "${VCS_BASENAME}", - "VCS_UUID": "${VCS_UUID}", - "VCS_NUM": ${VCS_NUM}, - "VCS_DATE": "${VCS_DATE}", - "VCS_BRANCH":"${VCS_BRANCH}", - "VCS_TAG": "${VCS_TAG}", - "VCS_TICK": ${VCS_TICK}, - "VCS_EXTRA": "${VCS_EXTRA}", - - "VCS_ACTION_STAMP": "${VCS_ACTION_STAMP}", - "VCS_FULL_HASH": "${VCS_FULL_HASH}", - "VCS_SHORT_HASH": "${VCS_SHORT_HASH}", - - "VCS_WC_MODIFIED": ${VCS_WC_MODIFIED} -} -EOF -} - -# For Java output -javaOutput() { - case "${VCS_WC_MODIFIED}" in - 1) VCS_WC_MODIFIED="true" ;; - 0) VCS_WC_MODIFIED="false" ;; - esac - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ - -public class autorevision { - public static final String VCS_TYPE = "${VCS_TYPE}"; - public static final String VCS_BASENAME = "${VCS_BASENAME}"; - public static final String VCS_UUID = "${VCS_UUID}"; - public static final long VCS_NUM = ${VCS_NUM}; - public static final String VCS_DATE = "${VCS_DATE}"; - public static final String VCS_BRANCH = "${VCS_BRANCH}"; - public static final String VCS_TAG = "${VCS_TAG}"; - public static final long VCS_TICK = ${VCS_TICK}; - public static final String VCS_EXTRA = "${VCS_EXTRA}"; - - public static final String VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"; - public static final String VCS_FULL_HASH = "${VCS_FULL_HASH}"; - public static final String VCS_SHORT_HASH = "${VCS_SHORT_HASH}"; - - public static final boolean VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; -} -EOF -} - -# For Java properties output -javapropOutput() { - case "${VCS_WC_MODIFIED}" in - 1) VCS_WC_MODIFIED="true" ;; - 0) VCS_WC_MODIFIED="false" ;; - esac - cat > "${TARGETFILE}" << EOF -# ${GENERATED_HEADER} - -VCS_TYPE=${VCS_TYPE} -VCS_BASENAME=${VCS_BASENAME} -VCS_UUID=${VCS_UUID} -VCS_NUM=${VCS_NUM} -VCS_DATE=${VCS_DATE} -VCS_BRANCH=${VCS_BRANCH} -VCS_TAG=${VCS_TAG} -VCS_TICK=${VCS_TICK} -VCS_EXTRA=${VCS_EXTRA} - -VCS_ACTION_STAMP=${VCS_ACTION_STAMP} -VCS_FULL_HASH=${VCS_FULL_HASH} -VCS_SHORT_HASH=${VCS_SHORT_HASH} - -VCS_WC_MODIFIED=${VCS_WC_MODIFIED} -EOF -} - -# For m4 output -m4Output() { - cat > "${TARGETFILE}" << EOF -dnl ${GENERATED_HEADER} -define(\`VCS_TYPE', \`${VCS_TYPE}')dnl -define(\`VCS_BASENAME', \`${VCS_BASENAME}')dnl -define(\`VCS_UUID', \`${VCS_UUID}')dnl -define(\`VCS_NUM', \`${VCS_NUM}')dnl -define(\`VCS_DATE', \`${VCS_DATE}')dnl -define(\`VCS_BRANCH', \`${VCS_BRANCH}')dnl -define(\`VCS_TAG', \`${VCS_TAG}')dnl -define(\`VCS_TICK', \`${VCS_TICK}')dnl -define(\`VCS_EXTRA', \`${VCS_EXTRA}')dnl -define(\`VCS_ACTIONSTAMP', \`${VCS_ACTION_STAMP}')dnl -define(\`VCS_FULLHASH', \`${VCS_FULL_HASH}')dnl -define(\`VCS_SHORTHASH', \`${VCS_SHORT_HASH}')dnl -define(\`VCS_WC_MODIFIED', \`${VCS_WC_MODIFIED}')dnl -EOF -} - -# For (La)TeX output -texOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - cat > "${TARGETFILE}" << EOF -% ${GENERATED_HEADER} -\def \vcsType {${VCS_TYPE}} -\def \vcsBasename {${VCS_BASENAME}} -\def \vcsUUID {${VCS_UUID}} -\def \vcsNum {${VCS_NUM}} -\def \vcsDate {${VCS_DATE}} -\def \vcsBranch {${VCS_BRANCH}} -\def \vcsTag {${VCS_TAG}} -\def \vcsTick {${VCS_TICK}} -\def \vcsExtra {${VCS_EXTRA}} -\def \vcsACTIONSTAMP {${VCS_ACTION_STAMP}} -\def \vcsFullHash {${VCS_FULL_HASH}} -\def \vcsShortHash {${VCS_SHORT_HASH}} -\def \vcsWCModified {${VCS_WC_MODIFIED}} -\endinput -EOF -} - -# For scheme output -schemeOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="#f" ;; - 1) VCS_WC_MODIFIED="#t" ;; - esac - cat > "${TARGETFILE}" << EOF -;; ${GENERATED_HEADER} -(define VCS_TYPE "${VCS_TYPE}") -(define VCS_BASENAME "${VCS_BASENAME}") -(define VCS_UUID "${VCS_UUID}") -(define VCS_NUM ${VCS_NUM}) -(define VCS_DATE "${VCS_DATE}") -(define VCS_BRANCH "${VCS_BRANCH}") -(define VCS_TAG "${VCS_TAG}") -(define VCS_TICK ${VCS_TICK}) -(define VCS_EXTRA "${VCS_EXTRA}") - -(define VCS_ACTION_STAMP "${VCS_ACTION_STAMP}") -(define VCS_FULL_HASH "${VCS_FULL_HASH}") -(define VCS_SHORT_HASH "${VCS_SHORT_HASH}") - -(define VCS_WC_MODIFIED ${VCS_WC_MODIFIED}) -;; end -EOF -} - -# For clojure output -clojureOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="false" ;; - 1) VCS_WC_MODIFIED="true" ;; - esac - cat > "${TARGETFILE}" << EOF -;; ${GENERATED_HEADER} -(def VCS_TYPE "${VCS_TYPE}") -(def VCS_BASENAME "${VCS_BASENAME}") -(def VCS_UUID "${VCS_UUID}") -(def VCS_NUM ${VCS_NUM}) -(def VCS_DATE "${VCS_DATE}") -(def VCS_BRANCH "${VCS_BRANCH}") -(def VCS_TAG "${VCS_TAG}") -(def VCS_TICK ${VCS_TICK}) -(def VCS_EXTRA "${VCS_EXTRA}") - -(def VCS_ACTION_STAMP "${VCS_ACTION_STAMP}") -(def VCS_FULL_HASH "${VCS_FULL_HASH}") -(def VCS_SHORT_HASH "${VCS_SHORT_HASH}") - -(def VCS_WC_MODIFIED ${VCS_WC_MODIFIED}) -;; end -EOF -} - -# For rpm spec file output -rpmOutput() { - cat > "${TARGETFILE}" << EOF -# ${GENERATED_HEADER} -$([ "${VCS_TYPE}" ] && echo "%define vcs_type ${VCS_TYPE}") -$([ "${VCS_BASENAME}" ] && echo "%define vcs_basename ${VCS_BASENAME}") -$([ "${VCS_UUID}" ] && echo "%define vcs_uuid ${VCS_UUID}") -$([ "${VCS_NUM}" ] && echo "%define vcs_num ${VCS_NUM}") -$([ "${VCS_DATE}" ] && echo "%define vcs_date ${VCS_DATE}") -$([ "${VCS_BRANCH}" ] && echo "%define vcs_branch ${VCS_BRANCH}") -$([ "${VCS_TAG}" ] && echo "%define vcs_tag ${VCS_TAG}") -$([ "${VCS_TICK}" ] && echo "%define vcs_tick ${VCS_TICK}") -$([ "${VCS_EXTRA}" ] && echo "%define vcs_extra ${VCS_EXTRA}") - -$([ "${VCS_ACTION_STAMP}" ] && echo "%define vcs_action_stamp ${VCS_ACTION_STAMP}") -$([ "${VCS_FULL_HASH}" ] && echo "%define vcs_full_hash ${VCS_FULL_HASH}") -$([ "${VCS_SHORT_HASH}" ] && echo "%define vcs_short_hash ${VCS_SHORT_HASH}") - -$([ "${VCS_WC_MODIFIED}" ] && echo "%define vcs_wc_modified ${VCS_WC_MODIFIED}") -# end -EOF -} - -# shellcheck disable=SC2155,SC2039 -hppOutput() { - local NAMESPACE="$(echo "${VCS_BASENAME}" | sed -e 's:_::g' | tr '[:lower:]' '[:upper:]')" - cat > "${TARGETFILE}" << EOF -/* ${GENERATED_HEADER} */ - -#ifndef ${NAMESPACE}_AUTOREVISION_H -#define ${NAMESPACE}_AUTOREVISION_H - -#include - -namespace $(echo "${NAMESPACE}" | tr '[:upper:]' '[:lower:]') -{ - const std::string VCS_TYPE = "${VCS_TYPE}"; - const std::string VCS_BASENAME = "${VCS_BASENAME}"; - const std::string VCS_UUID = "${VCS_UUID}"; - const int VCS_NUM = ${VCS_NUM}; - const std::string VCS_DATE = "${VCS_DATE}"; - const std::string VCS_BRANCH = "${VCS_BRANCH}"; - const std::string VCS_TAG = "${VCS_TAG}"; - const int VCS_TICK = ${VCS_TICK}; - const std::string VCS_EXTRA = "${VCS_EXTRA}"; - - const std::string VCS_ACTION_STAMP = "${VCS_ACTION_STAMP}"; - const std::string VCS_FULL_HASH = "${VCS_FULL_HASH}"; - const std::string VCS_SHORT_HASH = "${VCS_SHORT_HASH}"; - - const int VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; -} - -#endif - -/* end */ -EOF -} - -matlabOutput() { - case "${VCS_WC_MODIFIED}" in - 0) VCS_WC_MODIFIED="FALSE" ;; - 1) VCS_WC_MODIFIED="TRUE" ;; - esac - cat > "${TARGETFILE}" << EOF -% ${GENERATED_HEADER} - -VCS_TYPE = '${VCS_TYPE}'; -VCS_BASENAME = '${VCS_BASENAME}'; -VCS_UUID = '${VCS_UUID}'; -VCS_NUM = ${VCS_NUM}; -VCS_DATE = '${VCS_DATE}'; -VCS_BRANCH = '${VCS_BRANCH}'; -VCS_TAG = '${VCS_TAG}'; -VCS_TICK = ${VCS_TICK}; -VCS_EXTRA = '${VCS_EXTRA}'; - -VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}'; -VCS_FULL_HASH = '${VCS_FULL_HASH}'; -VCS_SHORT_HASH = '${VCS_SHORT_HASH}'; - -VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; - -% end -EOF -} - -octaveOutput() { - cat > "${TARGETFILE}" << EOF -% ${GENERATED_HEADER} - -VCS_TYPE = '${VCS_TYPE}'; -VCS_BASENAME = '${VCS_BASENAME}'; -VCS_UUID = '${VCS_UUID}'; -VCS_NUM = ${VCS_NUM}; -VCS_DATE = '${VCS_DATE}'; -VCS_BRANCH = '${VCS_BRANCH}'; -VCS_TAG = '${VCS_TAG}'; -VCS_TICK = ${VCS_TICK}; -VCS_EXTRA = '${VCS_EXTRA}'; - -VCS_ACTION_STAMP = '${VCS_ACTION_STAMP}'; -VCS_FULL_HASH = '${VCS_FULL_HASH}'; -VCS_SHORT_HASH = '${VCS_SHORT_HASH}'; - -VCS_WC_MODIFIED = ${VCS_WC_MODIFIED}; - -% end -EOF -} - -cmakeOutput() { - cat > "${TARGETFILE}" << EOF -# ${GENERATED_HEADER} - -set(VCS_TYPE ${VCS_TYPE}) -set(VCS_BASENAME ${VCS_BASENAME}) -set(VCS_UUID ${VCS_UUID}) -set(VCS_NUM ${VCS_NUM}) -set(VCS_DATE ${VCS_DATE}) -set(VCS_BRANCH ${VCS_BRANCH}) -set(VCS_TAG ${VCS_TAG}) -set(VCS_TICK ${VCS_TICK}) -set(VCS_EXTRA ${VCS_EXTRA}) - -set(VCS_ACTION_STAMP ${VCS_ACTION_STAMP}) -set(VCS_FULL_HASH ${VCS_FULL_HASH}) -set(VCS_SHORT_HASH ${VCS_SHORT_HASH}) - -set(VCS_WC_MODIFIED ${VCS_WC_MODIFIED}) - -# end -EOF -} - - -# Helper functions -# Count path segments -# shellcheck disable=SC2039 -pathSegment() { - local pathz="${1}" - local depth="0" - - if [ ! -z "${pathz}" ]; then - # Continue until we are at / or there are no path separators left. - while [ ! "${pathz}" = "/" ] && [ ! "${pathz}" = "$(echo "${pathz}" | sed -e 's:/::')" ]; do - pathz="$(dirname "${pathz}")" - depth="$((depth+1))" - done - fi - echo "${depth}" -} - -# Largest of four numbers -# shellcheck disable=SC2039 -multiCompare() { - local larger="${1}" - local numA="${2}" - local numB="${3}" - local numC="${4}" - - [ "${numA}" -gt "${larger}" ] && larger="${numA}" - [ "${numB}" -gt "${larger}" ] && larger="${numB}" - [ "${numC}" -gt "${larger}" ] && larger="${numC}" - echo "${larger}" -} - -# Test for repositories -# shellcheck disable=SC2155,SC2039 -repoTest() { - REPONUM="0" - if [ ! -z "$(git rev-parse HEAD 2>/dev/null)" ]; then - local gitPath="$(git rev-parse --show-toplevel)" - local gitDepth="$(pathSegment "${gitPath}")" - REPONUM="$((REPONUM+1))" - else - local gitDepth="0" - fi - if [ ! -z "$(hg root 2>/dev/null)" ]; then - local hgPath="$(hg root 2>/dev/null)" - local hgDepth="$(pathSegment "${hgPath}")" - REPONUM="$((REPONUM+1))" - else - local hgDepth="0" - fi - if [ ! -z "$(bzr root 2>/dev/null)" ]; then - local bzrPath="$(bzr root 2>/dev/null)" - local bzrDepth="$(pathSegment "${bzrPath}")" - REPONUM="$((REPONUM+1))" - else - local bzrDepth="0" - fi - if [ ! -z "$(svn info 2>/dev/null)" ]; then - local stringz="" - local stringx="" - local svnPath="$(svn info --xml | sed -n -e "s:${stringz}::" -e "s:${stringx}::p")" - # An old enough svn will not be able give us a path; default - # to 1 for that case. - if [ -z "${svnPath}" ]; then - local svnDepth="1" - else - local svnDepth="$(pathSegment "${svnPath}")" - fi - REPONUM="$((REPONUM+1))" - else - local svnDepth="0" - fi - - # Do not do more work then we have to. - if [ "${REPONUM}" = "0" ]; then - return - fi - - # Figure out which repo is the deepest and use it. - local wonRepo="$(multiCompare "${gitDepth}" "${hgDepth}" "${bzrDepth}" "${svnDepth}")" - if [ "${wonRepo}" = "${gitDepth}" ]; then - gitRepo - elif [ "${wonRepo}" = "${hgDepth}" ]; then - hgRepo - elif [ "${wonRepo}" = "${bzrDepth}" ]; then - bzrRepo - elif [ "${wonRepo}" = "${svnDepth}" ]; then - svnRepo - fi -} - - - -# Detect which repos we are in and gather data. -# shellcheck source=/dev/null -if [ -f "${CACHEFILE}" ] && [ "${CACHEFORCE}" = "1" ]; then - # When requested only read from the cache to populate our symbols. - . "${CACHEFILE}" -else - # If a value is not set through the environment set VCS_EXTRA to nothing. - : "${VCS_EXTRA:=""}" - repoTest - - if [ -f "${CACHEFILE}" ] && [ "${REPONUM}" = "0" ]; then - # We are not in a repo; try to use a previously generated cache to populate our symbols. - . "${CACHEFILE}" - # Do not overwrite the cache if we know we are not going to write anything new. - CACHEFORCE="1" - elif [ "${REPONUM}" = "0" ]; then - echo "error: No repo or cache detected." 1>&2 - exit 1 - fi -fi - - -# -s output is handled here. -if [ ! -z "${VAROUT}" ]; then - if [ "${VAROUT}" = "VCS_TYPE" ]; then - echo "${VCS_TYPE}" - elif [ "${VAROUT}" = "VCS_BASENAME" ]; then - echo "${VCS_BASENAME}" - elif [ "${VAROUT}" = "VCS_NUM" ]; then - echo "${VCS_NUM}" - elif [ "${VAROUT}" = "VCS_DATE" ]; then - echo "${VCS_DATE}" - elif [ "${VAROUT}" = "VCS_BRANCH" ]; then - echo "${VCS_BRANCH}" - elif [ "${VAROUT}" = "VCS_TAG" ]; then - echo "${VCS_TAG}" - elif [ "${VAROUT}" = "VCS_TICK" ]; then - echo "${VCS_TICK}" - elif [ "${VAROUT}" = "VCS_FULL_HASH" ]; then - echo "${VCS_FULL_HASH}" - elif [ "${VAROUT}" = "VCS_SHORT_HASH" ]; then - echo "${VCS_SHORT_HASH}" - elif [ "${VAROUT}" = "VCS_WC_MODIFIED" ]; then - echo "${VCS_WC_MODIFIED}" - elif [ "${VAROUT}" = "VCS_ACTION_STAMP" ]; then - echo "${VCS_ACTION_STAMP}" - else - echo "error: Not a valid output symbol." 1>&2 - exit 1 - fi -fi - - -# Detect requested output type and use it. -if [ ! -z "${AFILETYPE}" ]; then - if [ "${AFILETYPE}" = "c" ]; then - cOutput - elif [ "${AFILETYPE}" = "h" ]; then - hOutput - elif [ "${AFILETYPE}" = "xcode" ]; then - xcodeOutput - elif [ "${AFILETYPE}" = "swift" ]; then - swiftOutput - elif [ "${AFILETYPE}" = "sh" ]; then - shOutput - elif [ "${AFILETYPE}" = "py" ] || [ "${AFILETYPE}" = "python" ]; then - pyOutput - elif [ "${AFILETYPE}" = "pl" ] || [ "${AFILETYPE}" = "perl" ]; then - plOutput - elif [ "${AFILETYPE}" = "lua" ]; then - luaOutput - elif [ "${AFILETYPE}" = "php" ]; then - phpOutput - elif [ "${AFILETYPE}" = "ini" ]; then - iniOutput - elif [ "${AFILETYPE}" = "js" ]; then - jsOutput - elif [ "${AFILETYPE}" = "json" ]; then - jsonOutput - elif [ "${AFILETYPE}" = "java" ]; then - javaOutput - elif [ "${AFILETYPE}" = "javaprop" ]; then - javapropOutput - elif [ "${AFILETYPE}" = "tex" ]; then - texOutput - elif [ "${AFILETYPE}" = "m4" ]; then - m4Output - elif [ "${AFILETYPE}" = "scheme" ]; then - schemeOutput - elif [ "${AFILETYPE}" = "clojure" ]; then - clojureOutput - elif [ "${AFILETYPE}" = "rpm" ]; then - rpmOutput - elif [ "${AFILETYPE}" = "hpp" ]; then - hppOutput - elif [ "${AFILETYPE}" = "matlab" ]; then - matlabOutput - elif [ "${AFILETYPE}" = "octave" ]; then - octaveOutput - elif [ "${AFILETYPE}" = "cmake" ]; then - cmakeOutput - else - echo "error: Not a valid output type." 1>&2 - exit 1 - fi -fi - - -# If requested, make a cache file. -if [ ! -z "${CACHEFILE}" ] && [ ! "${CACHEFORCE}" = "1" ]; then - TARGETFILE="${CACHEFILE}.tmp" - shOutput - - # Check to see if there have been any actual changes. - if [ ! -f "${CACHEFILE}" ]; then - mv -f "${CACHEFILE}.tmp" "${CACHEFILE}" - elif cmp -s "${CACHEFILE}.tmp" "${CACHEFILE}"; then - rm -f "${CACHEFILE}.tmp" - else - mv -f "${CACHEFILE}.tmp" "${CACHEFILE}" - fi -fi diff --git a/ci/get-nprocessors.sh b/ci/get-nprocessors.sh deleted file mode 100644 index 43635e7..0000000 --- a/ci/get-nprocessors.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 Google Inc. -# All Rights Reserved. -# -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This file is typically sourced by another script. -# if possible, ask for the precise number of processors, -# otherwise take 2 processors as reasonable default; see -# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization -if [ -x /usr/bin/getconf ]; then - NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN) -else - NPROCESSORS=2 -fi - -# as of 2017-09-04 Travis CI reports 32 processors, but GCC build -# crashes if parallelized too much (maybe memory consumption problem), -# so limit to 4 processors for the time being. -if [ $NPROCESSORS -gt 4 ] ; then - echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4." - NPROCESSORS=4 -fi diff --git a/ci/perpare_pulp3_netrc.sh b/ci/perpare_pulp3_netrc.sh deleted file mode 100644 index 8414bbb..0000000 --- a/ci/perpare_pulp3_netrc.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -set -evx -echo "machine ${PULP3_SERVER_URL}\nlogin ${PULP3_SERVER_LOGIN}\npassword ${PULP3_SERVER_PASSWORD}\n" > ~/.netrc diff --git a/ci/travis.sh b/ci/travis.sh deleted file mode 100644 index 6e9056b..0000000 --- a/ci/travis.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env sh -set -evx - -chmod +x ci/get-nprocessors.sh -. ci/get-nprocessors.sh - -# if possible, ask for the precise number of processors, -# otherwise take 2 processors as reasonable default; see -# https://docs.travis-ci.com/user/speeding-up-the-build/#Makefile-optimization -if [ -x /usr/bin/getconf ]; then - NPROCESSORS=$(/usr/bin/getconf _NPROCESSORS_ONLN) -else - NPROCESSORS=2 -fi - -# as of 2017-09-04 Travis CI reports 32 processors, but GCC build -# crashes if parallelized too much (maybe memory consumption problem), -# so limit to 4 processors for the time being. -if [ $NPROCESSORS -gt 4 ] ; then - echo "$0:Note: Limiting processors to use by make from $NPROCESSORS to 4." - NPROCESSORS=4 -fi - -# Tell make to use the processors. No preceding '-' required. -MAKEFLAGS="j${NPROCESSORS}" -export MAKEFLAGS - -env | sort - -# Set default values to OFF for these variables if not specified. -: "${NO_EXCEPTION:=OFF}" -: "${NO_RTTI:=OFF}" -: "${COMPILER_IS_GNUCXX:=OFF}" - -# Install dependency from YUM -if [ -n "${INSTALL_DEPENDENCY_LIBRARY}" ]; then - yum install -y $INSTALL_DEPENDENCY_LIBRARY - source /etc/profile.d/framework.sh -fi - -if [ $ASAN_OPTION ] && [ -f "/opt/rh/devtoolset-7/enable" ] ;then - source /opt/rh/devtoolset-7/enable -fi - -mkdir build || true -cd build - -cmake3 -DCMAKE_CXX_FLAGS=$CXX_FLAGS \ - -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ - -DASAN_OPTION=$ASAN_OPTION \ - -DVERSION_DAILY_BUILD=$TESTING_VERSION_BUILD \ - .. - -make - -if [ -n "${PACKAGE}" ]; then - make package -fi - - -if [ -n "${UPLOAD}" ]; then - cp ~/rpm_upload_tools.py ./ - python3 rpm_upload_tools.py ${PULP3_REPO_NAME} ${PULP3_DIST_NAME} *.rpm -fi - -if [ -n "${UPLOAD_SYMBOL_FILES}" ]; then - rpm -i $SYMBOL_TARGET*debuginfo*.rpm - _symbol_file=`find /usr/lib/debug/ -name "$SYMBOL_TARGET*.so*.debug"` - cp $_symbol_file ${_symbol_file}info.${CI_COMMIT_SHORT_SHA} - sentry-cli upload-dif -t elf ${_symbol_file}info.${CI_COMMIT_SHORT_SHA} -fi diff --git a/cmake/Package.cmake b/cmake/Package.cmake deleted file mode 100644 index e112c59..0000000 --- a/cmake/Package.cmake +++ /dev/null @@ -1,57 +0,0 @@ -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - set(MY_RPM_NAME_PREFIX "lib${lib_name}-debug") -else() - set(MY_RPM_NAME_PREFIX "lib${lib_name}") -endif() - -message(STATUS "Package: ${MY_RPM_NAME_PREFIX}") - -set(CPACK_PACKAGE_VECDOR "MESA") -set(CPACK_PACKAGE_VERSION_MAJOR "${VERSION_MAJOR}") -set(CPACK_PACKAGE_VERSION_MINOR "${VERSION_MINOR}") -set(CPACK_PACKAGE_VERSION_PATCH "${VERSION_PATCH}.${VERSION_BUILD}") -set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) -set(CPACK_PACKAGE_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_BUILD}") - -execute_process(COMMAND sh changelog.sh ${CMAKE_BINARY_DIR} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/cmake) -SET(CPACK_RPM_CHANGELOG_FILE ${CMAKE_BINARY_DIR}/changelog.txt) - -# RPM Build -set(CPACK_GENERATOR "RPM") -set(CPACK_RPM_AUTO_GENERATED_FILE_NAME ON) -set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") -set(CPACK_RPM_PACKAGE_VENDOR "MESA") -set(CPACK_RPM_PACKAGE_AUTOREQPROV "yes") -set(CPACK_RPM_PACKAGE_RELEASE_DIST "on") -set(CPACK_RPM_DEBUGINFO_PACKAGE "on") - -set(CPACK_RPM_COMPONENT_INSTALL ON) -set(CPACK_COMPONENTS_IGNORE_GROUPS 1) -set(CPACK_COMPONENTS_GROUPING ONE_PER_GROUP) -set(CPACK_COMPONENT_HEADER_DISPLAY_NAME "develop") - -set(CPACK_COMPONENT_LIBRARIES_REQUIRED TRUE) -set(CPACK_RPM_LIBRARIES_PACKAGE_NAME ${MY_RPM_NAME_PREFIX}) -set(CPACK_COMPONENT_LIBRARIES_GROUP "LIBRARIES") - -set(CPACK_COMPONENT_HEADER_REQUIRED TRUE) -set(CPACK_RPM_HEADER_PACKAGE_NAME "${MY_RPM_NAME_PREFIX}-devel") -set(CPACK_COMPONENT_HEADER_GROUP "HEADER") - -set(CPACK_COMPONENT_TOOLS_REQUIRED TRUE) -set(CPACK_RPM_TOOLS_PACKAGE_NAME "${MY_RPM_NAME_PREFIX}-tools") -set(CPACK_COMPONENT_TOOLS_GROUP "TOOLS") - -set(CPACK_RPM_HEADER_PACKAGE_REQUIRES_PRE ${CPACK_RPM_LIBRARIES_PACKAGE_NAME}) -set(CPACK_RPM_HEADER_PACKAGE_CONFLICTS ${CPACK_RPM_HEADER_PACKAGE_NAME}) - -set(CPACK_COMPONENTS_ALL LIBRARIES HEADER TOOLS) - - -set(CPACK_BUILD_SOURCE_DIRS "${CMAKE_SOURCE_DIR}") - -# Must uninstall the debug package before install release package -set(CPACK_RPM_PACKAGE_CONFLICTS ${MY_RPM_NAME_PREFIX}) - -# set(CPACK_STRIP_FILES TRUE) -include(CPack) diff --git a/cmake/Version.cmake b/cmake/Version.cmake deleted file mode 100644 index a47944c..0000000 --- a/cmake/Version.cmake +++ /dev/null @@ -1,54 +0,0 @@ - -# Using autorevision.sh to generate version information - -set(__SOURCE_AUTORESIVISION ${CMAKE_SOURCE_DIR}/autorevision.sh) -set(__AUTORESIVISION ${CMAKE_BINARY_DIR}/autorevision.sh) -set(__VERSION_CACHE ${CMAKE_BINARY_DIR}/version.txt) -set(__VERSION_CONFIG ${CMAKE_BINARY_DIR}/version.cmake) - -file(COPY ${__SOURCE_AUTORESIVISION} DESTINATION ${CMAKE_BINARY_DIR} - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE) - -# execute autorevision.sh to generate version information -execute_process(COMMAND ${__AUTORESIVISION} -t cmake -o ${__VERSION_CACHE} - OUTPUT_FILE ${__VERSION_CONFIG} ERROR_QUIET) -include(${__VERSION_CONFIG}) - -# extract major, minor, patch version from git tag -string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" VERSION_MAJOR "${VCS_TAG}") -string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" VERSION_MINOR "${VCS_TAG}") -string(REGEX REPLACE "^v[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VCS_TAG}") -string(REGEX REPLACE "[T\\:\\+\\-]" "" VERSION_DATE "${VCS_DATE}") - -if(VERSION_DAILY_BUILD) - set(VERSION_PATCH ${VERSION_PATCH}.${VERSION_DATE}) -endif() - -if(NOT VERSION_MAJOR) - set(VERSION_MAJOR 1) -endif() - -if(NOT VERSION_MINOR) - set(VERSION_MINOR 0) -endif() - -if(NOT VERSION_PATCH) - set(VERSION_PATCH 0) -endif() - -set(VERSION "${VERSION_MAJOR}_${VERSION_MINOR}_${VERSION_PATCH}") -set(VERSION_BUILD "${VCS_SHORT_HASH}") - -# print information -message(STATUS "Version: ${VERSION}-${VERSION_BUILD}") - -option(DEFINE_GIT_VERSION "Set DEFINE_GIT_VERSION to TRUE or FALSE" TRUE) - -if(DEFINE_GIT_VERSION) - set(GIT_VERSION - "${VERSION}-${CMAKE_BUILD_TYPE}-${VERSION_BUILD}-${VCS_BRANCH}-${VCS_TAG}-${VCS_DATE}") - string(REGEX REPLACE "[-:+/\\.]" "_" GIT_VERSION ${GIT_VERSION}) - - add_definitions(-DGIT_VERSION=${GIT_VERSION}) -endif() diff --git a/cmake/changelog.sh b/cmake/changelog.sh deleted file mode 100644 index 9e5a277..0000000 --- a/cmake/changelog.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -work_path=$1 -branch=`git status | grep branch | awk '{print $NF}'` -git log --branches=$branch --no-merges --date=local --show-signature --pretty="** %cd %an %ae %nhash: %H%ncommit:%n%B" | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/ -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' | sed 's/\*/ -/g' | sed 's/ - -/\*/g' > $work_path/changelog.txt - -#git log --branches=$branch --no-merges --date=local --show-signature --pretty="** %cd %an %ae %nhash: %H%ncommit:%n%B" | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/ -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' > $work_path/changelog.txt diff --git a/inc/Maat_command.h b/inc/Maat_command.h deleted file mode 100644 index 3b783a4..0000000 --- a/inc/Maat_command.h +++ /dev/null @@ -1,221 +0,0 @@ -#ifndef H_MAAT_COMMAND_H_INCLUDE -#define H_MAAT_COMMAND_H_INCLUDE -#ifdef __cplusplus -extern "C"{ -#endif -#include "Maat_rule.h" -enum MAAT_OPERATION -{ - MAAT_OP_DEL=0, - MAAT_OP_ADD, - MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after -}; -enum MAAT_GROUP_RELATION -{ - PARENT_TYPE_COMPILE=0, - PARENT_TYPE_GROUP -}; -enum MAAT_REGION_TYPE -{ - REGION_EXPR, - REGION_IP, - REGION_IP_PLUS, - REGION_INTERVAL, - REGION_DIGEST, - REGION_SIMILARITY -}; -enum MAAT_EXPR_TYPE -{ - EXPR_TYPE_STRING=0, - EXPR_TYPE_AND, - EXPR_TYPE_REGEX, - EXPR_TYPE_OFFSET -}; -enum MAAT_MATCH_METHOD -{ - MATCH_METHOD_SUB=0, - MATCH_METHOD_SUFFIX, - MATCH_METHOD_PREFIX, - MATCH_METHOD_COMPLETE -}; - -enum MAAT_CASE_TYPE -{ - UNCASE_PLAIN=0, - CASE_HEXBIN, - CASE_PLAIN -}; -enum MAAT_ADDR_TYPE -{ - ADDR_TYPE_IPv4=4, - ADDR_TYPE_IPv6=6 -}; -enum MAAT_ADDR_DIRECTION -{ - ADDR_DIR_DOUBLE=0, - ADDR_DIR_SINGLE=1 -}; -struct Maat_rgn_str_t -{ - const char *keywords; - const char *district;// optional for expr_plus, otherwise set to NULL. - enum MAAT_EXPR_TYPE expr_type; - enum MAAT_MATCH_METHOD match_method; - enum MAAT_CASE_TYPE hex_bin; -}; -struct Maat_rgn_ip_t -{ - enum MAAT_ADDR_TYPE addr_type; - const char* src_ip; - const char* mask_src_ip; - const char* dst_ip; - const char* mask_dst_ip; - unsigned short src_port; - unsigned short mask_src_port; - unsigned short dst_port; - unsigned short mask_dst_port; - unsigned short protocol; - enum MAAT_ADDR_DIRECTION direction; -}; -struct Maat_rgn_ip_plus_t -{ - enum MAAT_ADDR_TYPE addr_type; - - const char* saddr_format;//mask, range or CIDR - const char* src_ip1; - const char* src_ip2; - const char* sport_format;//mask or range - unsigned short src_port1; - unsigned short src_port2; - - const char* daddr_format;//mask, range or CIDR - const char* dst_ip1; - const char* dst_ip2; - const char* dport_format;//mask or range - unsigned short dst_port1; - unsigned short dst_port2; - - unsigned short protocol; - enum MAAT_ADDR_DIRECTION direction; -}; -struct Maat_rgn_intv_t -{ - const char *district;// optional for expr_plus, otherwise set to NULL. - unsigned int low_boundary; - unsigned int up_boundary; -}; -struct Maat_rgn_digest_t -{ - unsigned long long orgin_len; - const char* digest_string; - short confidence_degree; -}; -struct Maat_rgn_sim_t -{ - char* target; - short threshold;// 1~100 -}; -struct Maat_region_t -{ - const char* table_name; - int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number. - enum MAAT_REGION_TYPE region_type; - union - { - struct Maat_rgn_str_t expr_rule; - struct Maat_rgn_ip_t ip_rule; - struct Maat_rgn_intv_t interval_rule; - struct Maat_rgn_digest_t digest_rule; - struct Maat_rgn_sim_t similarity_rule; - }; -}; -struct Maat_cmd_region -{ - const char* table_name; - int region_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must appoint a unique number. - enum MAAT_REGION_TYPE region_type; - union - { - struct Maat_rgn_str_t expr_rule; - struct Maat_rgn_ip_t ip_rule; - struct Maat_rgn_ip_plus_t ip_plus_rule; - struct Maat_rgn_intv_t interval_rule; - struct Maat_rgn_digest_t digest_rule; - struct Maat_rgn_sim_t similarity_rule; - }; -}; -struct Maat_cmd_line -{ - const char* table_name; - const char* table_line; - int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. - int label_id; - int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. -}; - -//Input string of REGION_EXPR and REGION_SIMILARITY need to be escapeed. -char* Maat_str_escape(char* dst,int size,const char*src); - - -//Returns number of successfully updated rule. -//Return -1 for failed. -int Maat_cmd_set_line(Maat_feather_t feather,const struct Maat_cmd_line* line_rule, enum MAAT_OPERATION op); -int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op); -int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, size_t size, enum MAAT_OPERATION op); - -//Return the value of key after the increment. -//If the key does not exist, it is set to 0 before performing the operation. -long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment); -struct Maat_cmd_key -{ - char* table_name; - int rule_id; -}; -void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number); -int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys); -int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size); -int Maat_cmd_flushDB(Maat_feather_t feather); - -struct Maat_cmd_group2group -{ - const char* table_name; - int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. - int superior_group_id; -}; -struct Maat_cmd_group2compile -{ - const char* table_name; - const char* virtual_table_name; - int group_id; - int compile_id; - int clause_index; - int not_flag; -}; - -int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id); -int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g); -int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c); - -//@param expire_after: expired after $expire_after$ seconds, set to 0 for never timeout. -//@param label_id: bigger than 0 means this compile rule is to be indexed and quried by Maat_cmd_select; =0 not index -int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after); - -struct Maat_command_batch; -struct Maat_command_batch* Maat_command_batch_new(Maat_feather_t feather); - -int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id); -int Maat_command_batch_set_group2group(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g); -int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c); -int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after); -int Maat_command_batch_commit(struct Maat_command_batch* batch); - -int Maat_command_get_new_group_id(Maat_feather_t feather); -int Maat_command_get_new_region_id(Maat_feather_t feather); - -#ifdef __cplusplus -} //end extern"C" -#endif - - -#endif - diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h deleted file mode 100644 index ab54f22..0000000 --- a/inc/Maat_rule.h +++ /dev/null @@ -1,353 +0,0 @@ - -/* -*****************Maat Deep Packet Inspection Policy Framework******** -* Maat is the Goddess of truth and justice in ancient Egyptian concept. -* Her feather was the measure that determined whether the souls (considered -* to reside in the heart) of the departed would reach the paradise of afterlife -* successfully. -* Author: contact@zhengchao.io -* Version 2022-10-24 version 4.0.0 -********************************************************* -*/ -#ifndef H_MAAT_RULE_H_INCLUDE -#define H_MAAT_RULE_H_INCLUDE -#ifdef __cplusplus -extern "C"{ -#endif -#include -enum MAAT_CHARSET -{ - CHARSET_NONE=0, - CHARSET_GBK, - CHARSET_BIG5, - CHARSET_UNICODE, - CHARSET_UTF8, // 4 - CHARSET_BIN, //5 - CHARSET_UNICODE_ASCII_ESC, // Unicode Escape format, prefix backslash-u hex, e.g. "\u627;" - CHARSET_UNICODE_ASCII_ALIGNED,//Unicode Escape format, prefix backslash-u with 4 bytes aligned, e.g. "\u0627" - CHARSET_UNICODE_NCR_DEC, //SGML Numeric character reference,decimal base, e.g. "ا" - CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا" - CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA - CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD - CHARSET_WINDOWS1251, - __CHARSET_MAX -}; -enum MAAT_ACTION -{ - MAAT_ACTION_BLOCK=0, - MAAT_ACTION_MONIT, - MAAT_ACTION_WHITE -}; -enum MAAT_POS_TYPE -{ - MAAT_POSTYPE_EXPR=0, - MAAT_POSTYPE_REGEX -}; -typedef void* scan_status_t; -typedef void* stream_para_t; -typedef void* Maat_feather_t; - - -#define MAX_SERVICE_DEFINE_LEN 128 -#define MAX_HUGE_SERVICE_DEFINE_LEN (1024*4) -struct Maat_rule_t -{ - int config_id; - int service_id; - unsigned char do_log; - unsigned char do_blacklist; - unsigned char action; - unsigned char reserved; - int serv_def_len; - char service_defined[MAX_SERVICE_DEFINE_LEN]; -}; -#define MAAT_RULE_UPDATE_TYPE_FULL 1 -#define MAAT_RULE_UPDATE_TYPE_INC 2 -typedef void Maat_start_callback_t(int update_type,void* u_para); -typedef void Maat_update_callback_t(int table_id,const char* table_line,void* u_para); -typedef void Maat_finish_callback_t(void* u_para); - - - - - -//--------------------HITTING DETAIL DESCRIPTION BEGIN - -#define MAAT_MAX_HIT_RULE_NUM 8 -#define MAAT_MAX_EXPR_ITEM_NUM 8 -#define MAAT_MAX_HIT_POS_NUM 8 -#define MAAT_MAX_REGEX_GROUP_NUM 8 - -//NOTE position buffer as hitting_regex_pos and hit_pos,are ONLY valid before next scan or Maat_stream_scan_string_end -struct regex_pos_t -{ - int group_num; - int hitting_regex_len; - const char* hitting_regex_pos; - int grouping_len[MAAT_MAX_REGEX_GROUP_NUM]; - const char* grouping_pos[MAAT_MAX_REGEX_GROUP_NUM]; -}; -struct str_pos_t -{ - int hit_len; - const char* hit_pos; -}; -struct sub_item_pos_t -{ - enum MAAT_POS_TYPE ruletype; - int hit_cnt; - union - { - struct regex_pos_t regex_pos[MAAT_MAX_HIT_POS_NUM]; - struct str_pos_t substr_pos[MAAT_MAX_HIT_POS_NUM]; - }; -}; - -struct Maat_region_pos_t -{ - - int region_id; - int sub_item_num; - struct sub_item_pos_t sub_item_pos[MAAT_MAX_EXPR_ITEM_NUM]; -}; - -struct Maat_hit_detail_t -{ - int config_id;//set <0 if half hit; - int hit_region_cnt; - struct Maat_region_pos_t region_pos[MAAT_MAX_HIT_RULE_NUM]; -}; -//--------------------HITTING DETAIL DESCRIPTION END - -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather(int max_thread_num, - const char* table_info_path, - const char* ful_cfg_dir, - const char* inc_cfg_dir, - void*logger);//MESA_handle_logger -//Abondon interface ,left for compatible. -Maat_feather_t Maat_summon_feather_json(int max_thread_num, - const char* table_info_path, - const char* json_rule, - void* logger); - -Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger); -int Maat_initiate_feather(Maat_feather_t feather); - -enum MAAT_INIT_OPT -{ - MAAT_OPT_SCANDIR_INTERVAL_MS=1, //VALUE is interger, SIZE=sizeof(int). DEFAULT:1,000 milliseconds. - MAAT_OPT_EFFECT_INVERVAL_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:60,000 milliseconds. - MAAT_OPT_FULL_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_INC_CFG_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_JSON_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default. - MAAT_OPT_STAT_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. - MAAT_OPT_PERF_ON, //VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF. - MAAT_OPT_STAT_FILE_PATH, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT: no default. - MAAT_OPT_SCAN_DETAIL, //VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return hit pos, not include regex grouping. - // 2 return hit pos and regex grouping pos;DEFAULT:0 - MAAT_OPT_INSTANCE_NAME, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1, no more than 11 bytes.DEFAULT: MAAT_$tableinfo_path$. - MAAT_OPT_DECRYPT_KEY, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT. - MAAT_OPT_REDIS_IP, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT. - MAAT_OPT_REDIS_PORT, //VALUE is a unsigned short or a signed int, host order, SIZE= sizeof(unsigned short) or sizeof(int). No DEFAULT. - MAAT_OPT_REDIS_INDEX, //VALUE is interger *, 0~15, SIZE=sizeof(int). DEFAULT: 0. - MAAT_OPT_CMD_AUTO_NUMBERING, //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1. - MAAT_OPT_DEFERRED_LOAD, //VALUE is NULL,SIZE is 0. Default: Deffered initialization OFF. - MAAT_OPT_CUMULATIVE_UPDATE_OFF, //VALUE is NULL,SIZE is 0. Default: CUMMULATIVE UPDATE ON. - MAAT_OPT_LOAD_VERSION_FROM, //VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old. - //This option also disables background update. - MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. - MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} - MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files - MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS, //VALUE is interger, SIZE=sizeof(int). DEFAULT:10,000 milliseconds. - MAAT_OPT_JSON_IS_GZIPPED, //VALUE is NULL, SIZE is 0. Default: 0, Not compressed by gzip. - MAAT_OPT_STATUS_OUTPUT_PROMETHEUS //VALUE is a interger *, 1 or 0, SIZE=sizeof(int). DEFAULT: 1. -}; -//return -1 if failed, return 0 on success; -int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); -enum MAAT_STATE_OPT -{ - MAAT_STATE_VERSION=1, //Get current maat version, if maat is in update progress, the updating version is returned. VALUE is long long, SIZE=sizeof(long long). - MAAT_STATE_LAST_UPDATING_TABLE, //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no - MAAT_STATE_IN_UPDATING -}; -int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size); - -void Maat_burn_feather(Maat_feather_t feather); - -//return table_id(>=0) if success,otherwise return -1; -int Maat_table_register(Maat_feather_t feather,const char* table_name); -//return 1 if success,otherwise return -1 incase invalid table_id or registed function number exceed 32; -int Maat_table_callback_register(Maat_feather_t feather,short table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para); - -#define MAX_HIT_REGION_NUM_PER_GROUP 128 - -struct Maat_hit_path_t -{ - int Nth_scan; - int region_id; - int sub_group_id; - int top_group_id; - int virtual_table_id; // 0 is not a virtual table. - int compile_id; -}; -enum MAAT_SCAN_OPT -{ - MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. - MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. - MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N, - //Maat_get_scan_status returns actual got number. -}; -//return 0 if success, return -1 when failed; -int Maat_set_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, const void* value, int size); - -//return >=0 if success, return -1 when failed; -int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size); - -//Return hit rule number, return -1 when error occurs,return -2 when hit current region -//mid MUST set to NULL before fist call -int Maat_scan_intval(Maat_feather_t feather,int table_id - ,unsigned int intval - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_scan_proto_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr,unsigned short int proto - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num); -int Maat_full_scan_string(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid,int thread_num); -//hit_detail could be NULL if not cared. -int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid,int thread_num); - -stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num); -int Maat_stream_scan_string(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid); -//hited_detail could be NULL if not cared. -int Maat_stream_scan_string_detail(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid); -void Maat_stream_scan_string_end(stream_para_t* stream_para); - -stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num); -int Maat_stream_scan_digest(stream_para_t* stream_para - ,const char* data,int data_len,unsigned long long offset - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t* mid); -void Maat_stream_scan_digest_end(stream_para_t* stream_para); - -int Maat_similar_scan_string(Maat_feather_t feather,int table_id - ,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t* mid,int thread_num); - -void Maat_clean_status(scan_status_t* mid); - -typedef void* MAAT_RULE_EX_DATA; -// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered. -// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called. -typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, - MAAT_RULE_EX_DATA* ad, long argl, void *argp); -typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, - MAAT_RULE_EX_DATA* ad, long argl, void *argp); -typedef void Maat_rule_EX_dup_func_t(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp); - -int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, - Maat_rule_EX_new_func_t* new_func, - Maat_rule_EX_free_func_t* free_func, - Maat_rule_EX_dup_func_t* dup_func, - long argl, void *argp); -//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data. -MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx); - -//Sort rules by their evaluation order. -//rule_array will be modified with sorted rule. -//Return sortted rule number, maybe less than n_rule if some rules are invalid. -size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_rule_t* rule_array, size_t n_rule); - - -//Helper function for parsing space or tab seperated line. -//Nth_column: the Nth column is numberd from 1. -//Return 0 if success. -int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len); - - -//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. -typedef void* MAAT_PLUGIN_EX_DATA; -typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); -typedef int Maat_plugin_EX_key2index_func_t(const char* key); - -int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp); -//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to FREE the data. -MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); - - -int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - long argl, void *argp); - -struct ip_address -{ - int ip_type; //4: IPv4, 6: IPv6 - union - { - unsigned int ipv4; //network order - unsigned int ipv6[4]; - }; -}; - -int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); - -int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - long argl, void *argp); - -//Return order: Longger suffix first, then fqdn with bigger index first. -int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); - -int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t *new_func, - Maat_plugin_EX_free_func_t *free_func, - Maat_plugin_EX_dup_func_t *dup_func, - long argl, void *argp); -int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); - -enum MAAT_RULE_OPT -{ - MAAT_RULE_SERV_DEFINE //VALUE is a char* buffer,SIZE= buffer size. -}; -int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size); - -#ifdef __cplusplus -}//end extern "C" -#endif - - -#endif // H_MAAT_RULE_H_INCLUDE - diff --git a/inc/bool_matcher.h b/inc/bool_matcher.h deleted file mode 100644 index 25a0554..0000000 --- a/inc/bool_matcher.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * - * Copyright (c) 2018 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2021-06-12 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef INCLUDE_BOOL_MATCHER_H -#define INCLUDE_BOOL_MATCHER_H -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - #define MAX_ITEMS_PER_BOOL_EXPR 8 - - /* not_flag=0表示布尔项item_id必须出现；not_flag=1表示布尔项item_idä¸èƒ½å‡ºçŽ° */ - struct bool_item - { - unsigned long long item_id; - unsigned char not_flag; - }; - - /* At least one item's not_flag should be 0. */ - struct bool_expr - { - unsigned long long expr_id; - void * user_tag; - size_t item_num; - struct bool_item items[MAX_ITEMS_PER_BOOL_EXPR]; - }; - - struct bool_expr_match - { - unsigned long long expr_id; - void * user_tag; - }; - - struct bool_matcher; - - struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size); - - /* Returned results are sorted by expr_id in descending order. */ - //Input item_ids MUST be ASCENDING order and NO duplication. - int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result); - - void bool_matcher_free(struct bool_matcher * matcher); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/inc/gram_index_engine.h b/inc/gram_index_engine.h deleted file mode 100644 index 452dc4d..0000000 --- a/inc/gram_index_engine.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef _GRAM_INDEX_ENGINE_ -#define _GRAM_INDEX_ENGINE_ - -#ifdef __cplusplus -extern "C" { -#endif -enum GIE_operation -{ - GIE_INSERT_OPT, - GIE_DELETE_OPT -}; -enum GIE_INPUT_FORMAT -{ - GIE_INPUT_FORMAT_PLAIN, - GIE_INPUT_FORMAT_SFH -}; - -typedef struct -{ - /* data */ -}GIE_handle_t; - - -typedef struct -{ - unsigned int id; - unsigned int sfh_length;//size of fuzzy_hash - enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; - short cfds_lvl; - char * sfh; - void * tag; -}GIE_digest_t; - - -typedef struct -{ - unsigned int id; - short cfds_lvl; - void * tag; -}GIE_result_t; - - -typedef struct -{ - unsigned int gram_value; - //unsigned int htable_num; - unsigned int position_accuracy; - enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string - //else id format==PALIN, means the input string is common string - int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result -}GIE_create_para_t; - - -GIE_handle_t * GIE_create(const GIE_create_para_t * para); - - -int GIE_update(GIE_handle_t * handle, GIE_digest_t ** digests, int size); - - -//return actual matched result count -//return 0 when matched nothing; -//return -1 when error occurs; -int GIE_query(GIE_handle_t * handle, const char * data, int data_len, GIE_result_t * results, int result_size); - -void GIE_destory(GIE_handle_t * handle); -int GIE_string_similiarity(const char *str1, int len1, const char *str2, int len2); -int GIE_sfh_similiarity(const char *sfh1, int len1, const char *sfh2, int len2); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/inc/stream_fuzzy_hash.h b/inc/stream_fuzzy_hash.h deleted file mode 100644 index 9e85e81..0000000 --- a/inc/stream_fuzzy_hash.h +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef _STREAM_FUZZY_HASH_ -#define _STREAM_FUZZY_HASH_ - -/* - * Copyright (C) MESA 2015 - - * - */ - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define TOTAL_LENGTH 0 -#define EFFECTIVE_LENGTH 1 -#define HASH_LENGTH 2 - -// typedef sfh_instance_t void*; -typedef struct -{ -}sfh_instance_t; - -/** - * create a fuzzy hash handle and return it. - * @return [handle] - */ -sfh_instance_t * SFH_instance(unsigned long long origin_len); - -/** - * destroy context by a fuzzy hash handle. - * @param handle [handle] - */ -void SFH_release(sfh_instance_t * handle); - -/** - * Feed the function your data. - * Call this function several times, if you have several parts of data to feed. - * @param handle [handle] - * @param data [data that you want to fuzzy_hash] - * @param size [data size] - * @param offset [offset] - * @return [return effective data length in current feed] - */ -unsigned int SFH_feed(sfh_instance_t * handle, const char* data, unsigned int size, unsigned long long offset); - -/** - * Obtain the fuzzy hash values. - * @param handle [handle] - * @param result [fuzzy hash result] - * Fuzzy hash result with offsets(in the square brackets, with colon splitted). - * eg. abc[1:100]def[200:300] - * @param size [@result size] - * @return [return zero on success, non-zero on error] - */ -int SFH_digest(sfh_instance_t * handle, char* result, unsigned int size); - -/** - * Obtain certain length of fuzzy hash status. - * @param handle [handle] - * @param type [length type] - * TOTAL_LENGTH:Total length of data you have fed. - * Overlapped data will NOT count for 2 times. - * EFFECTIVE_LENGTH:Length of data that involved in the calculation of hash. - * HASH_LENGTH:Hash result length. - * @return [length value] - */ -unsigned long long SFH_status(sfh_instance_t * handle, int type); - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/include/maat/maat.h b/include/maat/maat.h new file mode 100644 index 0000000..5d17528 --- /dev/null +++ b/include/maat/maat.h @@ -0,0 +1,23 @@ +#pragma once +struct maat_feather; +/* network-order */ +struct ipv4_tuple4{ + UINT32 saddr; /* network order */ + UINT32 daddr; /* network order */ + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; + +#define IPV6_ADDR_LEN (sizeof(struct in6_addr)) + +struct ipv6_tuple4 +{ + UCHAR saddr[IPV6_ADDR_LEN] ; + UCHAR daddr[IPV6_ADDR_LEN] ; + UINT16 source; /* network order */ + UINT16 dest; /* network order */ +}; +struct maat_scan_state; +int maat_scan_ipv4(struct maat_feather *feather, struct maat_scan_state *state, const struct ipv4_tuple4 *tuple4, unsigned int *matched_ids, size_t n_match_id); +int maat_scan_string(struct maat_feather *feather, struct maat_scan_state *state, const char *data, size_t length, ) + diff --git a/lib/.gitignore b/lib/.gitignore deleted file mode 100644 index 65043fb..0000000 --- a/lib/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -SI/ -*.I* -*.P* -*.S* -*.W* -*.[od] -*.[1-9]* -*.log -*.a diff --git a/readme.md b/readme.md deleted file mode 100644 index 0394427..0000000 --- a/readme.md +++ /dev/null @@ -1,25 +0,0 @@ -# 简介 -MAAT是å¤åŸƒåŠç¥žè¯ä¸­çœŸç†ä¸Žæ­£ä¹‰å¥³ç¥žï¼Œå¥¹çš„羽毛（feather）能够判断离世之人能å¦å‰å¾€å¤©å ‚。 - -MAAT框架对网络æµå¤„ç†ä¸­çš„é…置进行抽象，并具有é…置写入ã€å¤šæœºåŒæ­¥ï¼ˆåŸºäºŽRedis）ã€åŠ è½½ã€æ‰«æ的功能。 - -Maat_rule.h中æ述了åˆå§‹åŒ–å’Œé…置扫æçš„API。 - -Maat_command.h中æ述了é…置写入的API。 - -# 编译 -普通编译 - -` Make` - -带调试符å·ç¼–译 - -`Make debug` - -安装 - -`Make install` - -生æˆåŠ¨æ€é“¾æŽ¥åº“ `./build/src/libmaatframe.so` - -生æˆé™æ€é“¾æŽ¥åº“ `./build/src/libmaatframe.a` \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index b880656..0000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -cmake_minimum_required(VERSION 3.5) - -set(MAAT_FRAME_MAJOR_VERSION 3) -set(MAAT_FRAME_MINOR_VERSION 6) -set(MAAT_FRAME_PATCH_VERSION 0) -set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) - -message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") - -add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table_schema.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) -include_directories(/opt/MESA/include/MESA/) - -# Static Library Output -add_library(maat_frame_static STATIC ${MAAT_SRC}) -set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) -set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) -#target_include_directories(maat_frame_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) -target_link_libraries(maat_frame_static hiredis-static) -target_link_libraries(maat_frame_static igraph-static) -target_link_libraries(maat_frame_static ipmatcher-static) - -# Shared Library Output -add_library(maat_frame_shared SHARED ${MAAT_SRC}) -set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) -set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) -set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) -set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) -set_target_properties(maat_frame_shared PROPERTIES LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/version.map") -target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/) -target_include_directories(maat_frame_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/inc_internal/hiredis) -target_link_libraries(maat_frame_shared hiredis-static ${MAAT_DEPEND_DYN_LIB}) -target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB}) -target_link_libraries(maat_frame_shared ipmatcher-static) - -set(CMAKE_INSTALL_PREFIX /opt/MESA/) -install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) -install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_command.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) -install(FILES ${PROJECT_SOURCE_DIR}/inc/stream_fuzzy_hash.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) -install(FILES ${PROJECT_SOURCE_DIR}/inc/gram_index_engine.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) -install(FILES ${PROJECT_SOURCE_DIR}/inc/bool_matcher.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) -install(TARGETS maat_frame_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ COMPONENT LIBRARIES) diff --git a/src/entry/.gitignore b/src/entry/.gitignore deleted file mode 100644 index 65043fb..0000000 --- a/src/entry/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -SI/ -*.I* -*.P* -*.S* -*.W* -*.[od] -*.[1-9]* -*.log -*.a diff --git a/src/entry/FQDN_engine.cpp b/src/entry/FQDN_engine.cpp deleted file mode 100644 index d402f66..0000000 --- a/src/entry/FQDN_engine.cpp +++ /dev/null @@ -1,352 +0,0 @@ -/* - * - * Copyright (c) 2020 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2020-09-01 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#include "FQDN_engine.h" -#include -#include -#include -#include -#include - -/*************************************************************************************/ - -//#include -//#define popcnt_u64 _mm_popcnt_u64 -//Use gcc builtin function to replace SSE4.2 instruction for portability - -#ifdef _MSC_VER -#include -#define popcnt_u64 __popcnt -#else -#define popcnt_u64 __builtin_popcountl -#endif - -#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++) - -struct packedRT_t -{ - unsigned long long bitmap[4]; - unsigned int A; - unsigned char B[4]; -}; - -void * aligned_malloc(size_t size, size_t align) -{ - void * malloc_ptr; - void * aligned_ptr; - - /* Error if align is not a power of two. */ - if (align & (align - 1)) - { - return ((void*) 0); - } - - if (align==0 || size == 0) - { - return ((void *) 0); - } - - malloc_ptr = malloc (sizeof(void *) + align - 1 + size); - if (!malloc_ptr) - { - return ((void *) 0); - } - - aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1)); - - ((void **) aligned_ptr) [-1] = malloc_ptr; - - return aligned_ptr; -} - -void aligned_free(void * aligned_ptr) -{ - if (aligned_ptr) - { - free (((void **) aligned_ptr) [-1]); - } -} - -/*************************************************************************************/ -struct domain_impl_t -{ - unsigned int id; - int suf_match; - unsigned int len; - unsigned long long hash; /*ÓÃ64λ¹þϣֵΨһ±íʾһ¸öÓòÃû*/ - domain_impl_t * next; - void * utag; -}; - -class CHashTrieFQDN -{ -public: - CHashTrieFQDN(); - ~CHashTrieFQDN(); - - int initialize(const struct FQDN_rule * rules, size_t n_rule); - int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); - -protected: - unsigned int rank(unsigned int h); - -protected: - unsigned int m_num; - domain_impl_t * m_domains; - unsigned int m_H; - unsigned int m_max_pat_len; - unsigned long long * m_B; - packedRT_t * m_RT; - domain_impl_t ** m_matched; - unsigned char m_case_tab[256]; -}; - - -const unsigned long long ONE=1; - -#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) ) -#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) ) - -const unsigned long long A=6364136223846793005LL; - -CHashTrieFQDN::CHashTrieFQDN() -{ - m_num=0; - m_domains=NULL; - m_B=NULL; - m_RT=NULL; - m_matched=NULL; - FOR(c, 256) m_case_tab[c]=tolower(c); -} - -CHashTrieFQDN::~CHashTrieFQDN() -{ - if(m_domains!=NULL) - { - delete [] m_domains; - } - - if(m_B!=NULL) - { - delete [] m_B; - } - - if(m_RT!=NULL) - { - aligned_free(m_RT); - } - - if(m_matched!=NULL) - { - delete [] m_matched; - } -} - -int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule) -{ - long long mem_bytes=0; - - if(n_rule==0) return -1; - m_num=n_rule; - m_domains=new domain_impl_t[m_num]; - mem_bytes+=m_num*sizeof(domain_impl_t); - - unsigned int N=m_num; - m_max_pat_len=0; - - FOR(k, m_num) - { - m_domains[k].id =rules[k].id; - m_domains[k].suf_match=rules[k].is_suffix_match; - m_domains[k].len=rules[k].len; - m_domains[k].next=NULL; - m_domains[k].utag=rules[k].user_tag; - - FOR(j, rules[k].len) - { - if(rules[k].FQDN[j]=='.') ++N; - } - - if(m_max_pat_len>6]; - mem_bytes+=(m_H>>6)*sizeof(unsigned long long); - FOR(i, (m_H>>6)) m_B[i]=0; - - m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64); - mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t); - FOR(i, (m_H>>8)) - { - FOR(j, 4) m_RT[i].bitmap[j]=0; - } - - FOR(k, m_num) - { - const unsigned char * pb=(const unsigned char *)rules[k].FQDN; - m_domains[k].hash=0; - - for(int j=rules[k].len-1; j>=-1; --j) - { - if(j==-1 || pb[j]=='.') - { - unsigned int h=m_domains[k].hash&(m_H-1); - set_bit(m_B, h); - if(j==-1) - { - int q=h&255; - m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63)); - } - } - else - { - m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]]; - } - } - } - - m_RT[0].A=0; - FOR(i, (m_H>>8)) - { - m_RT[i].B[0]=0; - m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]); - m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]); - m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]); - m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]); - } - - int tn=m_RT[m_H>>8].A; - - m_matched=new domain_impl_t *[tn]; - mem_bytes+=tn*sizeof(domain_impl_t *); - FOR(i, tn) m_matched[i]=NULL; - - FOR(k, m_num) - { - unsigned int h=m_domains[k].hash&(m_H-1); - unsigned idx=rank(h); - m_domains[k].next=m_matched[idx]; - m_matched[idx]=&(m_domains[k]); - } - -// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20)); - - return 1; -} - -unsigned int CHashTrieFQDN::rank(unsigned int h) -{ - int p=(h>>8); - int r=((h&255)>>6); - int s=(h&63); - unsigned long long e=m_RT[p].bitmap[r]&((ONE<=-1; --j) - { - if(j==-1 || pb[j]=='.') - { - unsigned int h=hash&(m_H-1); - if(is_bit_set(m_B, h)==0) break; - HASH[t]=hash; - P[t]=j+1; - ++t; - } - else if(j+m_max_pat_len=0; t--) - { - unsigned int h=HASH[t]&(m_H-1); - int q=h&255; - - if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63))) - { - unsigned idx=rank(h); - - for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next) - { - if(P[t]!=0 && pt->suf_match==0) continue; - if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t]) - { - //if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num; - results[match_num].id=pt->id; - results[match_num].offset=P[t]; - results[match_num].user_tag=pt->utag; - ++match_num; - if(match_num==n_result) return match_num; - } - } - } - } - - return match_num; -} - -/*************************************************************************************/ -struct FQDN_engine -{ - CHashTrieFQDN ht; -}; - -struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule) -{ - struct FQDN_engine * instance=new struct FQDN_engine; - if(instance->ht.initialize(rules, n_rule)<0) - { - delete instance; - return NULL; - } - else - { - return instance; - } -} - -int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result) -{ - if(instance==NULL) return -1; - return instance->ht.search(FQDN, FQDN_len, results, n_result); -} - -void FQDN_engine_free(struct FQDN_engine * instance) -{ - if(instance!=NULL) delete instance; -} diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp deleted file mode 100644 index ce226bd..0000000 --- a/src/entry/Maat_api.cpp +++ /dev/null @@ -1,2667 +0,0 @@ -#include -#include -#include -#include -#include - -#include "rulescan.h" -#include "bool_matcher.h" -#include "Maat_rule.h" -#include "Maat_rule_internal.h" -#include "Maat_utils.h" -#include "Maat_garbage_collection.h" -#include "Maat_hierarchy.h" -#include "dynamic_array.h" -#include "alignment_int64.h" -#include "config_monitor.h" -#include "map_str2int.h" -#include "rulescan.h" -#include "json2iris.h" - - -inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) -{ - alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); - return; -} -inline void DEC_SCANNER_REF(Maat_scanner*scanner,int thread_num) -{ - - alignment_int64_array_add(scanner->ref_cnt, thread_num, -1); - return; -} - -inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid) -{ - if( _mid && - _mid->is_last_scan==1 && - _mid->compile_mid && - Maat_hierarchy_compile_mid_has_NOT_clause(_mid->compile_mid)) - { - return 1; - } - else - { - return 0; - } -} - - -//return 1 if insert a unique id -//return 0 if id is duplicated -//return -1 if set is full -int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned long long id) -{ - size_t i=0; - for(i=0; iscan_cnt++; - return; -} - -void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len) -{ - memcpy(rule, rule_head, sizeof(struct Maat_rule_head)); - memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); - return; -} -struct compile_sort_para -{ - double evaluation_order; - int declared_clause_num; - int compile_id; - void* user; -}; -static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_rule* compile_relation, void* user) -{ - para->compile_id=compile_relation->compile_id; - para->evaluation_order=compile_relation->evaluation_order; - para->declared_clause_num=compile_relation->declared_clause_num; - para->user=user; - return; -} -static int compile_sort_para_compare(const struct compile_sort_para* a, const struct compile_sort_para* b) -{ - //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority. - if(a->evaluation_order!=0 && b->evaluation_order!=0) - { - if(a->evaluation_order - b->evaluation_order <0) - { - return -1; - } - else if(a->evaluation_order - b->evaluation_order >0) - { - return 1; - } - } - //If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority. - else if(a->evaluation_order + b->evaluation_order!= 0) - { - return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1; - } - //If compile rule's execute sequences are not specified or equal. - if(a->declared_clause_num!=b->declared_clause_num) - { - return (a->declared_clause_num-b->declared_clause_num); - } - else - { - return (b->compile_id-a->compile_id); - } - -} -static int compare_compile_rule(const void *a, const void *b) -{ - const struct Maat_compile_rule *ra=*(const struct Maat_compile_rule **)a; - const struct Maat_compile_rule *rb=*(const struct Maat_compile_rule **)b; - - struct compile_sort_para sa, sb; - compile_sort_para_set(&sa, ra, NULL); - compile_sort_para_set(&sb, rb, NULL); - - return compile_sort_para_compare(&sa, &sb); -} -static int compile_sort_para_compare_no_type(const void* a, const void* b) -{ - return compile_sort_para_compare((const struct compile_sort_para*) a, (const struct compile_sort_para*) b); - -} -size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_rule_t* rule_array, size_t n_rule) -{ - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct compile_sort_para sort_para[n_rule]; - struct Maat_rule_t copy_rule_array[n_rule]; - - memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule); - - struct Maat_compile_rule *p=NULL; - - size_t i=0, j=0; - for(i=0; iscanner->hier, rule_array[i].config_id); - if(p) - { - compile_sort_para_set(sort_para+j, p, copy_rule_array+j); - j++; - } - } - qsort(sort_para, j, sizeof(struct compile_sort_para), - compile_sort_para_compare_no_type); - for(i=0; in_wrapped_region=0; - wraper->virtual_table_id=0; - wraper->virtual_table_ids=NULL; - for(i=0; i< n_rslt; i++) - { - region=(struct Maat_region_inner*)(rulescan_rslt[i].tag); - if(region->district_id==district_id||district_id==DISTRICT_ANY) - { - wraper->wrapped_regions[wraper->n_wrapped_region]=region; - wraper->n_wrapped_region++; - } - } - wraper->is_last_region=is_last_region; - wraper->virtual_table_id=virtual_table_id; - wraper->Nth_scan=Nth_scan; - wraper->virtual_table_ids=NULL; - return; -} -void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* wraper, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virtual_table_id, int Nth_scan) -{ - size_t i=0; - wraper->n_wrapped_region=0; - wraper->virtual_table_id=0; - wraper->virtual_table_ids=NULL; - for(i=0; i< n_rslt; i++) - { - wraper->wrapped_regions[wraper->n_wrapped_region]=(struct Maat_region_inner*)(GIE_rslt[i].tag); - } - wraper->n_wrapped_region=n_rslt; - wraper->is_last_region=is_last_region; - wraper->virtual_table_id=virtual_table_id; - wraper->Nth_scan=Nth_scan; - wraper->virtual_table_ids=NULL; - return; -} - -int region_compile(_Maat_feather_t*feather, struct Maat_hierarchy_compile_mid* compile_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result, int size,int thread_num) -{ - int is_last_region=region_hit_wraper->is_last_region; - size_t region_hit_num=region_hit_wraper->n_wrapped_region; - - int scan_ret=0; - int i=0; - - struct Maat_compile_rule* compile_rule_array[size]; - struct Maat_compile_rule* compile_rule=NULL; - int virtual_table_id=0; - - struct Maat_region_inner* region=NULL; - - for(i=0; (size_t)iwrapped_regions[i]; - assert(region->magic_num==REGION_RULE_MAGIC); - if(region_hit_wraper->virtual_table_ids) - { - virtual_table_id=region_hit_wraper->virtual_table_ids[i]; - } - else - { - virtual_table_id=region_hit_wraper->virtual_table_id; - } - Maat_hierarchy_compile_mid_udpate(feather->scanner->hier, compile_mid, region->region_id, virtual_table_id, region_hit_wraper->Nth_scan, i); - - } - scan_ret=Maat_hierarchy_region_compile(feather->scanner->hier, compile_mid, is_last_region, (void**)compile_rule_array, size); - //Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule. - if(scan_ret>1) - { - qsort(compile_rule_array, scan_ret, sizeof(struct Maat_compile_rule*), - compare_compile_rule); - } - for(i=0; imagic_num==COMPILE_RULE_MAGIC); - - fill_maat_rule(&(result[i]), &(compile_rule->head), - compile_rule->service_defined, compile_rule->head.serv_def_len); - - } - - if(scan_ret>0) - { - alignment_int64_array_add(feather->hit_cnt, thread_num, 1); - } - if(region_hit_num==0&&scan_ret>0) - { - alignment_int64_array_add(feather->not_grp_hit_cnt, thread_num, 1); - - } - return MIN(scan_ret, size); -} - - -int fill_regex_pos(struct regex_pos_t *regex_pos,int size,rule_result_t *rs_result,const char* buff) -{ - int i=0,j=0; - int group_num=rs_result->group_num; - unsigned int * position=rs_result->position; - unsigned int * length=rs_result->length; - regex_pos->group_num=group_num; - for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM) - { - break; - } - regex_pos[i].hitting_regex_pos=buff+position[(group_num+1)*i]; - regex_pos[i].hitting_regex_len=length[(group_num+1)*i]; - for(j=0;jMAAT_MAX_REGEX_GROUP_NUM) - { - break; - } - regex_pos[i].grouping_pos[j]=buff+position[(group_num+1)*i+j+1]; - regex_pos[i].grouping_len[j]=length[(group_num+1)*i+j+1]; - } - } - return i; -} -int fill_substr_pos(struct str_pos_t* str_pos,int size,rule_result_t *rs_result,const char* buff) -{ - int i=0; - unsigned int * position=rs_result->position; - unsigned int * length=rs_result->length; - for(i=0;i<(int)rs_result->result_num&&iMAX_MATCH_POS_NUM) - { - break; - } - str_pos[i].hit_pos=buff+position[i]; - str_pos[i].hit_len= length[i]; - } - return i; -} -int hit_pos_RS2Maat(struct sub_item_pos_t* maat_sub_item,int size,rule_result_t* rs_result,int rs_cnt,const char* buff) -{ - int k=0; - for(k=0;kNth_hit_region; - hit_detail[i].region_pos[j].region_id=hit_paths[j]->path.region_id; - hit_detail[i].region_pos[j].sub_item_num=region_hit[Nth_hit_region].rnum; - hit_pos_RS2Maat(hit_detail[i].region_pos[j].sub_item_pos, MAAT_MAX_EXPR_ITEM_NUM, - region_hit[Nth_hit_region].result, region_hit[Nth_hit_region].rnum, scan_buff); - r_in_c_flag[Nth_hit_region]=1; - } - hit_detail[i].hit_region_cnt=j; - } - //for each region_ids, if belongs to no compile cfg, fill hit_detail as a half hit cfg - for(j=0; j<(size_t)region_cnt && iregion_id; - hit_detail[i].region_pos[0].sub_item_num=region_hit[j].rnum; - hit_pos_RS2Maat(hit_detail[i].region_pos[0].sub_item_pos,MAAT_MAX_EXPR_ITEM_NUM, - region_hit[j].result, region_hit[j].rnum, scan_buff); - i++; - } - } - return i; - */ - return 0; -} - -struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num) -{ - struct _OUTER_scan_status_t* outer_mid=NULL; - outer_mid=ALLOC(struct _OUTER_scan_status_t, 1); - outer_mid->feather=feather; - outer_mid->district_id=DISTRICT_ANY; - outer_mid->thread_num=(signed short)thread_num; - return outer_mid; -} -struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid, _Maat_feather_t* feather, int thread_num, int is_hit_region) -{ - struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*raw_mid);; - if(_mid==NULL) - { - _mid=_make_outer_status(feather, thread_num); - *raw_mid=_mid; - if(_mid->thread_num>=0)//Maat_set_scan_status calls grap_mid() with thread_num=-1. - { - alignment_int64_array_add(feather->outer_mid_cnt, thread_num, 1); - } - } - if(_mid->thread_num<0 && thread_num>=0) - { - _mid->thread_num=thread_num; - alignment_int64_array_add(feather->outer_mid_cnt, thread_num, 1); - } - - if(is_hit_region==1) - { - if(_mid->compile_mid==NULL) - { - _mid->compile_mid=Maat_hierarchy_compile_mid_new(feather->scanner->hier, thread_num); - alignment_int64_array_add(feather->compile_mid_cnt, thread_num, 1); - } - } - return _mid; -} -int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,int data_len, long* cache_offset) -{ - int to_copy_size=0,foward_offset=0; - int ret_len; - if(data_len<=buff_size-detained_len) - { - to_copy_size=data_len; - memcpy(buff+detained_len,data,data_len); - ret_len=detained_len+data_len; - } - else if(data_len>buff_size-detained_len&&data_len=buff_size - { - memcpy(buff,data+data_len-buff_size,buff_size); - ret_len=buff_size; - *cache_offset+=(data_len-buff_size+detained_len); - } - return ret_len; -} -int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz) -{ - int ret=0; - struct stat fstat_buf; - unsigned char* json_buff=NULL, *decrypted_buff=NULL, *uncompressed_buff=NULL; - size_t json_buff_sz=0, decrypted_buff_sz=0, uncompressed_buff_sz=0; - - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module , - "Maat initial with JSON file %s, formating..", - maat_json_fn); - - if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) - { - ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, - "Decrypt Maat JSON file %s failed.", maat_json_fn); - return -1; - } - json_buff=decrypted_buff; - json_buff_sz=decrypted_buff_sz; - - } - if(feather->maat_json_is_gzipped) - { - ret=gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); - free(json_buff); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, - "Uncompress Maat JSON file %s failed.", maat_json_fn); - return -1; - } - json_buff=uncompressed_buff; - json_buff_sz=uncompressed_buff_sz; - } - if(json_buff==NULL)//decryption failed or no decryption. - { - ret=load_file_to_memory(maat_json_fn, &json_buff, &json_buff_sz); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger, RLOG_LV_FATAL, maat_module, - "Read Maat JSON file %s failed.", maat_json_fn); - return -1; - } - } - ret=json2iris((const char*)json_buff, - maat_json_fn, - feather->compile_tn, feather->group2compile_tn, feather->group2group_tn, - NULL, - feather->json_ctx.iris_file, - sizeof(feather->json_ctx.iris_file), - strlen(feather->decrypt_key)?feather->decrypt_key:NULL, - strlen(feather->decrypt_algo)?feather->decrypt_algo:NULL, - feather->logger); - free(json_buff); - json_buff=NULL; - if(ret<0) - { - return -1; - } - if(!feather->is_running) - { - strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file)); - } - - ret=stat(maat_json_fn, &fstat_buf); - feather->json_ctx.last_md5_time=fstat_buf.st_ctim; - - md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module, - "JSON file %s md5: %s, generate index file %s OK.", - feather->json_ctx.json_file, - feather->json_ctx.effective_json_md5, - feather->json_ctx.iris_file); - feather->input_mode=SOURCE_JSON_FILE; - - return 0; -} -Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) -{ - if(max_thread_num<=0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Invalid max_thread_num=%d." - ,max_thread_num); - return NULL; - } - _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_mgr=Maat_table_manager_create(table_info_path, logger); - if(feather->table_mgr==NULL) - { - goto failed; - } - Maat_table_manager_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); - Maat_table_manager_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn)); - Maat_table_manager_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn)); - - feather->logger=logger; - feather->scan_thread_num=max_thread_num; - - feather->rule_effect_interval_ms=60*1000; - feather->rule_update_checking_interval_ms=1*1000; - feather->garbage_collection_timeout_ms=10*1000; - feather->rule_scan_type=0; - feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num); - feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num); - feather->compile_mid_cnt=alignment_int64_array_alloc(max_thread_num); - feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); - feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num); - feather->maat_version=0; - feather->last_full_version=0; - feather->base_grp_seq=0; - feather->base_rgn_seq=0; - feather->AUTO_NUMBERING_ON=1; - feather->backgroud_update_enabled=1; - feather->maat_json_is_gzipped=0; - - snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); - snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); - pthread_mutex_init(&(feather->background_update_mutex),NULL); - snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); - return feather; -failed: - free(feather); - return NULL; -} -int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int intval=0,ret=-1; - char err_str[MAX_TABLE_NAME_LEN]; - switch(type) - { - case MAAT_OPT_ENABLE_UPDATE: - intval=*((int*)value); - if(_feather->backgroud_update_enabled!=intval) - { - if(intval==0) - { - pthread_mutex_lock(&(_feather->background_update_mutex)); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Background update is disabled, current version %lld." - ,_feather->maat_version); - } - else - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Background update is enabled, current version %lld." - ,_feather->maat_version); - } - _feather->backgroud_update_enabled=intval; - } - else - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Duplicated operation, background update is ALREADY %s, current version %lld." - ,_feather->backgroud_update_enabled?"enabled":"disabled" - ,_feather->maat_version); - - } - return 0; - default: - break; - } - if(_feather->is_running==1)//The following options are not allowed to set after initiation; - { - return -2; - } - switch(type) - { - case MAAT_OPT_EFFECT_INVERVAL_MS: - intval=*(const int*)value; - if(size!=sizeof(int)||intval<0) - { - return -1; - } - _feather->rule_effect_interval_ms=intval; - break; - case MAAT_OPT_SCANDIR_INTERVAL_MS: - intval=*(const int*)value; - if(size!=sizeof(int)||intval<0) - { - return -1; - } - _feather->rule_update_checking_interval_ms=intval; - break; - case MAAT_OPT_GARBAGE_COLLECTION_TIMEOUT_MS: - intval=*(const int*)value; - if(size!=sizeof(int)||intval<0) - { - return -1; - } - _feather->garbage_collection_timeout_ms=intval; - break; - case MAAT_OPT_STATUS_OUTPUT_PROMETHEUS: - intval=*(const int*)value; - if(size!=sizeof(int)||intval<0) - { - return -1; - } - _feather->output_prometheus=intval; - break; - case MAAT_OPT_JSON_IS_GZIPPED: - intval=*(const int*)value; - if(size!=sizeof(int)||intval<0) - { - return -1; - } - if(intval>0) - { - _feather->maat_json_is_gzipped=1; - } - else - { - _feather->maat_json_is_gzipped=0; - } - break; - case MAAT_OPT_FULL_CFG_DIR: - assert(_feather->input_mode==SOURCE_NONE); - if(size>(int)sizeof(_feather->iris_ctx.full_dir)) - { - return -1; - } - memcpy(_feather->iris_ctx.full_dir,(const char*)value,size); - _feather->input_mode=SOURCE_IRIS_FILE; - break; - case MAAT_OPT_INC_CFG_DIR: - if(size>(int)sizeof(_feather->iris_ctx.inc_dir)) - { - return -1; - } - memcpy(_feather->iris_ctx.inc_dir,(const char*)value,size); - break; - case MAAT_OPT_JSON_FILE_PATH: - assert(_feather->input_mode==SOURCE_NONE); - ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str)); - if(ret<0) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, - "Load maat json file %s failed: %s.", - (const char*)value, err_str); - return -1; - } - break; - case MAAT_OPT_STAT_ON: - _feather->stat_on=1; - break; - case MAAT_OPT_PERF_ON: - _feather->perf_on=1; - break; - case MAAT_OPT_STAT_FILE_PATH: - if(size>(int)sizeof(_feather->stat_file)) - { - return -1; - } - memcpy(_feather->stat_file, (const char*)value, size); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module, - "Maat performance statistic output to %s.", - (const char*)value); - _feather->stat_on=1; - break; - case MAAT_OPT_SCAN_DETAIL: - intval=*(const int*)value; - _feather->rule_scan_type=intval; - break; - case MAAT_OPT_INSTANCE_NAME: - snprintf(_feather->instance_name, - sizeof(_feather->instance_name), - "%s", - (const char*)value); - break; - case MAAT_OPT_DECRYPT_KEY: - if((size_t)size>sizeof(_feather->decrypt_key)) - { - return -1; - } - memcpy(_feather->decrypt_key,value,size); - break; - case MAAT_OPT_REDIS_IP: - assert(_feather->input_mode==SOURCE_NONE); - if((size_t)size>sizeof(_feather->mr_ctx.redis_ip)) - { - return -1; - } - memcpy(_feather->mr_ctx.redis_ip,value,size); - _feather->input_mode=SOURCE_REDIS; - break; - case MAAT_OPT_REDIS_PORT: - if((size_t)size==sizeof(unsigned short)) - { - _feather->mr_ctx.redis_port=*((unsigned short*)value); - } - else if((size_t)size==sizeof(int)) - { - _feather->mr_ctx.redis_port=*((int*)value); - } - else - { - return -1; - } - break; - case MAAT_OPT_REDIS_INDEX: - if((size_t)size!=sizeof(int)) - { - return -1; - } - _feather->mr_ctx.redis_db=*((int*)value); - break; - case MAAT_OPT_CMD_AUTO_NUMBERING: - if((size_t)size!=sizeof(int)||*((int*)value)>15||*((int*)value)<0) - { - return -1; - } - _feather->AUTO_NUMBERING_ON=*((int*)value); - break; - case MAAT_OPT_DEFERRED_LOAD: - _feather->DEFERRED_LOAD_ON=1; - break; - case MAAT_OPT_CUMULATIVE_UPDATE_OFF: - _feather->cumulative_update_off=1; - break; - case MAAT_OPT_LOAD_VERSION_FROM: - _feather->load_version_from=*((long long*)value); - _feather->backgroud_update_enabled=1; - pthread_mutex_lock((&_feather->background_update_mutex)); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat load version from %lld, backgroud update stopped." - ,_feather->load_version_from); - break; - case MAAT_OPT_ACCEPT_TAGS: - _feather->n_tags=parse_accept_tag((const char*) value, &_feather->accept_tags, _feather->logger); - if(_feather->n_tags==0) - { - return -1; - } - _feather->accept_tags_raw=_maat_strdup((const char*) value); - break; - case MAAT_OPT_FOREIGN_CONT_DIR: - memset(_feather->foreign_cont_dir, 0, sizeof(_feather->foreign_cont_dir)); - strncpy(_feather->foreign_cont_dir, (char*)value, sizeof(_feather->foreign_cont_dir)); - if(_feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]=='/') - { - _feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]='\0'; - } - break; - default: - return -1; - } - return 0; -} -void maat_read_full_config(_Maat_feather_t* _feather) -{ - struct source_redis_ctx* mr_ctx=NULL; - switch(_feather->input_mode) - { - case SOURCE_REDIS: - mr_ctx=&(_feather->mr_ctx); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initiate from Redis %s:%hu db%d." - ,mr_ctx->redis_ip - ,mr_ctx->redis_port - ,mr_ctx->redis_db); - mr_ctx->read_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, _feather->logger); - if(mr_ctx->read_ctx != NULL) - { - redis_monitor_traverse(_feather->maat_version, - mr_ctx, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - _feather, - _feather->decrypt_key, //Not used. - _feather); - } - if(_feather->update_tmp_scanner==NULL) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: no avilable rule in redis %s:%hu db%d", - mr_ctx->redis_ip, - mr_ctx->redis_port, - mr_ctx->redis_db); - } - break; - case SOURCE_IRIS_FILE: - config_monitor_traverse(_feather->maat_version, - _feather->iris_ctx.full_dir, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - _feather, - _feather->decrypt_key, - _feather->logger); - if(_feather->update_tmp_scanner==NULL) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: NO effective rule in %s.", - _feather->iris_ctx.full_dir); - } - break; - case SOURCE_JSON_FILE: - config_monitor_traverse(_feather->maat_version, - _feather->json_ctx.iris_file, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - _feather, - _feather->decrypt_key, - _feather->logger); - if(_feather->update_tmp_scanner==NULL) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: NO efffective rule in JSON generate %s.", - _feather->json_ctx.iris_file); - } - break; - default: - break; - } - _feather->scanner=_feather->update_tmp_scanner; - _feather->update_tmp_scanner=NULL; - _feather->is_running=1; - if(_feather->scanner!=NULL) - { - _feather->maat_version=_feather->scanner->version; - _feather->last_full_version=_feather->scanner->version; - } - return; -} -int Maat_initiate_feather(Maat_feather_t feather) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - system_cmd_mkdir(_feather->foreign_cont_dir); - _feather->garbage_bin=Maat_garbage_bin_new(_feather->rule_effect_interval_ms/1000+_feather->garbage_collection_timeout_ms/1000); - - if(_feather->DEFERRED_LOAD_ON==0) - { - maat_read_full_config(_feather); - } - if(strlen(_feather->stat_file)==0) - { - if(_feather->stat_on==1) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: MAAT_OPT_STAT_FILE_PATH not set, TURN OFF STAT trigger."); - } - _feather->stat_on=0; - } - if(_feather->stat_on==0&&_feather->perf_on==1) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: STAT tirigger OFF, TURN OFF PERF trigger."); - _feather->perf_on=0; - } - if(_feather->cumulative_update_off==1) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Update with cumulative version OFF."); - } - if(_feather->n_tags>0) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Accept tags: %s", _feather->accept_tags_raw); - } - if(_feather->stat_on==1) - { - maat_stat_init(_feather); - } - - pthread_create(&(_feather->cfg_mon_t), NULL, thread_rule_monitor, (void*)_feather); - return 0; - -} - -Maat_feather_t Maat_summon_feather(int max_thread_num, - const char* table_info_path, - const char* ful_cfg_dir, - const char* inc_cfg_dir, - void* logger) -{ - int ret=-1; - Maat_feather_t feather=NULL; - feather=Maat_feather(max_thread_num,table_info_path,logger); - if(feather==NULL) - { - return NULL; - } - ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); - if(ret<0) - { - goto error_out; - } - ret=Maat_set_feather_opt(feather, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); - if(ret<0) - { - goto error_out; - } - ret=Maat_initiate_feather(feather); - if(ret<0) - { - goto error_out; - } - return feather; - -error_out: - Maat_burn_feather(feather); - return NULL; - - -} -Maat_feather_t Maat_summon_feather_json(int max_thread_num, - const char* table_info_path, - const char* json_rule, - void* logger) -{ - int ret=-1; - Maat_feather_t feather=NULL; - feather=Maat_feather(max_thread_num,table_info_path,logger); - if(feather==NULL) - { - return NULL; - } - ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_rule, strlen(json_rule)+1); - if(ret<0) - { - goto error_out; - } - ret=Maat_initiate_feather(feather); - if(ret<0) - { - goto error_out; - } return feather; - -error_out: - - Maat_burn_feather(feather); - return NULL; - -} - -void Maat_burn_feather(Maat_feather_t feather) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - _feather->is_running=0;//destroy will proceed in thread_rule_monitor - void* ret=NULL; - pthread_join(_feather->cfg_mon_t, &ret); - return; -} -int Maat_table_register(Maat_feather_t feather,const char* table_name) -{ - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - return Maat_table_manager_get_id_by_name(_feather->table_mgr, table_name); -} -int Maat_table_callback_register(Maat_feather_t feather,short table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para) -{ - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int i=0,ret=0; - - pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_manager_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); - if(ret<0) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return -1; - } - if(!_feather->scanner) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 1; - } - const char* line=NULL; - struct Maat_table_runtime* table_rt=NULL; - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - long long line_cnt=Maat_table_runtime_plugin_cached_row_count(table_rt); - - if(line_cnt>0) - { - if(start!=NULL) - { - start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); - } - for(i=0; ibackground_update_mutex)); - return 1; -} - -void rule_ex_data_new_cb(void * user_data, void * param) -{ - struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)param; - struct Maat_compile_rule* compile=(struct Maat_compile_rule*)user_data; - MAAT_RULE_EX_DATA ad=NULL; - - if(compile->ref_table->table_id!=ex_desc->table_id) - { - return; - } - ad=rule_ex_data_new(&(compile->head), - compile->service_defined, - ex_desc); - compile->ads[ex_desc->idx]=ad; - return; -} -int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, - Maat_rule_EX_new_func_t *new_func, - Maat_rule_EX_free_func_t* free_func, - Maat_rule_EX_dup_func_t* dup_func, - long argl, void *argp) -{ - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int idx=-1; - - if(new_func==NULL || free_func==NULL || dup_func==NULL) - { - return -1; - } - pthread_mutex_lock(&(_feather->background_update_mutex)); - idx=Maat_table_manager_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, - new_func, - free_func, - dup_func, - argl, argp); - - if(idx<0) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return -1; - } - struct compile_ex_data_idx* compile_ex_desc=Maat_table_manager_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); - - if(_feather->scanner!=NULL) - { - Maat_hierarchy_compile_user_data_iterate(_feather->scanner->hier, rule_ex_data_new_cb, compile_ex_desc); - } - - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return idx; - -} -MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) -{ - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_rule* compile=NULL; - const struct compile_table_schema* compile_desc=NULL; - const struct compile_ex_data_idx* ex_desc=NULL; - MAAT_RULE_EX_DATA ad=NULL; - if(!_feather || !_feather->scanner) - { - return NULL; - } - compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id); - if(compile==NULL) - { - return NULL; - } - compile_desc=&(compile->ref_table->compile); - assert(idxex_data_num); - ex_desc=compile_desc->ex_desc+idx; - ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp); - return ad; -} - - -MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_schema *table_desc=NULL; - struct Maat_table_runtime *table_rt=NULL; - MAAT_RULE_EX_DATA exdata=NULL; - struct timespec start,end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - - if(_feather->scanner==NULL) - { - return NULL; - } - table_desc=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); - } - - return exdata; -} -int Maat_generic_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t *new_func, - Maat_plugin_EX_free_func_t *free_func, - Maat_plugin_EX_dup_func_t *dup_func, - long argl, void *argp) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=-1; - struct Maat_table_schema *table_schema=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id); - pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_schema_set_EX_data_schema(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); - - if(ret<0) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return -1; - } - - struct Maat_table_runtime* table_rt=NULL; - if(_feather->scanner!=NULL) - { - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_commit_EX_data_schema(table_rt, table_schema, _feather->logger); - } - pthread_mutex_unlock(&(_feather->background_update_mutex)); - - return 0; - -} -int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp) - -{ - int ret=-1; - ret=Maat_generic_plugin_EX_register(feather, table_id, - new_func, free_func, dup_func, - argl, argp); - return ret; - -} - -int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t *new_func, - Maat_plugin_EX_free_func_t *free_func, - Maat_plugin_EX_dup_func_t *dup_func, - long argl, void *argp) - -{ - int ret=-1; - ret=Maat_generic_plugin_EX_register(feather, table_id, - new_func, free_func, dup_func, - argl, argp); - return ret; -} -int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t *new_func, - Maat_plugin_EX_free_func_t *free_func, - Maat_plugin_EX_dup_func_t *dup_func, - long argl, void *argp) -{ - int ret=-1; - ret=Maat_generic_plugin_EX_register(feather, table_id, - new_func, free_func, dup_func, - argl, argp); - return ret; -} -int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t *new_func, - Maat_plugin_EX_free_func_t *free_func, - Maat_plugin_EX_dup_func_t *dup_func, - long argl, void *argp) -{ - int ret=-1; - ret=Maat_generic_plugin_EX_register(feather, table_id, - new_func, free_func, dup_func, - argl, argp); - return ret; -} - - -int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) -{ - struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; - struct Maat_table_schema *table_schema=NULL; - struct Maat_table_runtime *table_rt=NULL; - int n_get=0; - - struct timespec start, end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - if(_feather->scanner==NULL) - { - return 0; - } - - table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL); - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); - if(table_type!=TABLE_TYPE_IP_PLUGIN) - { - return -1; - } - struct ip_data ip_data=*(const struct ip_data*)ip; - if(ip_data.type==IPv4) - { - ip_data.ipv4=ntohl(ip_data.ipv4); - } - else - { - ipv6_ntoh(ip_data.ipv6); - } - n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data); - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); - } - - return n_get; - -} -int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) -{ - struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; - struct Maat_table_schema *table_schema=NULL; - struct Maat_table_runtime *table_rt=NULL; - int n_get=0; - - struct timespec start, end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - if(_feather->scanner==NULL) - { - return 0; - } - table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL); - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); - if(table_type!=TABLE_TYPE_FQDN_PLUGIN) - { - return -1; - } - n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data); - - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); - } - - return n_get; -} -int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) -{ - struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; - struct Maat_table_schema *table_schema=NULL; - struct Maat_table_runtime *table_rt=NULL; - int n_get=0; - - struct timespec start, end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - if(_feather->scanner==NULL) - { - return 0; - } - table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_BOOL_PLUGIN, NULL); - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); - if(table_type!=TABLE_TYPE_BOOL_PLUGIN) - { - return -1; - } - n_get=Maat_table_runtime_bool_plugin_get_N_ex_data(table_rt, table_schema, item_ids, n_item, ex_data_array, n_ex_data); - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); - } - - return n_get; -} - -int Maat_full_scan_string_detail(Maat_feather_t feather, int table_id, - enum MAAT_CHARSET charset, const char* data, int data_len, - struct Maat_rule_t*result, int rule_num, struct Maat_hit_detail_t *hit_detail, int detail_num, - int* detail_ret, scan_status_t* mid, int thread_num) -{ - int region_ret=0, compile_ret=0; - int district_id=DISTRICT_ANY; - size_t hit_region_cnt=0; - unsigned int sub_type=0; - int virtual_table_id=0; - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); - - scan_result_t *region_result=NULL; - struct Maat_table_schema *p_table=NULL; - struct string_table_schema* expr_desc=NULL; - struct timespec start,end; - Maat_scanner* my_scanner=NULL; - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - _mid=grab_mid(mid, _feather, thread_num, 0); - scan_staus_count_inc(_mid); - if(data==NULL||data_len<=0) - { - return -1; - } - my_scanner=_feather->scanner; - if(my_scanner==NULL) - { - return 0; - } - - p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); - if(p_table==NULL) - { - _feather->scan_err_cnt++; - return -1; - } - expr_desc=&(p_table->expr); - - if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) - { - _feather->scan_err_cnt++; - return -1; - } - if(expr_desc->do_charset_merge==1) - { - sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); - } - else - { - sub_type=make_sub_type(p_table->table_id, charset,0); - } - alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - scan_data_t scan_data; - scan_data.text_data.text=data; - scan_data.text_data.tlen=data_len; - scan_data.text_data.toffset=0; - - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); -// Even no rule in table, we still need to search for NOT compile. -// if(table_rt->origin_rule_num==0) -// { -// return 0; -// } - assert(thread_num<_feather->scan_thread_num); - region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; - - INC_SCANNER_REF(my_scanner, thread_num); - if(table_rt->expr.expr_rule_cnt>0) - { - scan_data.rule_type=RULETYPE_STR; - scan_data.sub_type=sub_type; - region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result, MAX_SCANNER_HIT_NUM); - if(region_ret>0) - { - hit_region_cnt+=region_ret; - } - } - if(table_rt->expr.regex_rule_cnt>0) - { - scan_data.rule_type=RULETYPE_REG; - scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); - region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); - if(region_ret>0) - { - hit_region_cnt+=region_ret; - } - - } - - if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) - { - struct scan_region_hit_wraper region_hit_wraper; - if(hit_region_cnt>0) - { - alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); - } - _mid=grab_mid(mid, _feather, thread_num, 1); - if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) - { - district_id=_mid->district_id; - } - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, district_id, - _mid->is_last_scan, virtual_table_id, _mid->scan_cnt); - - if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid)) - { - compile_ret=region_compile(_feather, _mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0) - { - *detail_ret=fill_region_hit_detail(data, _mid->compile_mid, - region_result, hit_region_cnt, - result, compile_ret, - _mid->scan_cnt, - hit_detail, detail_num, my_scanner); - } - } - } - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, thread_num); - } - DEC_SCANNER_REF(my_scanner, thread_num); - if(compile_ret==0&&hit_region_cnt>0) - { - return -2; - } - return compile_ret; -} -int Maat_full_scan_string(Maat_feather_t feather, int table_id, - enum MAAT_CHARSET charset, const char* data, int data_len, - struct Maat_rule_t*result, int* found_pos, int rule_num, - scan_status_t* mid, int thread_num) -{ - int detail_ret=0,compile_ret=0; - compile_ret=Maat_full_scan_string_detail(feather,table_id, - charset, data,data_len, - result, rule_num, - NULL, 0, &detail_ret,mid,thread_num); - return compile_ret; -} -int Maat_scan_intval(Maat_feather_t feather,int table_id - ,unsigned int intval - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num) -{ - int region_ret=0,compile_ret=0; - int district_id=DISTRICT_ANY; - struct _OUTER_scan_status_t* _mid=NULL; - scan_data_t intval_scan_data; - scan_result_t *region_result=NULL; - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner* my_scanner=NULL; - - Maat_table_schema* p_table=NULL; - struct timespec start,end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - _mid=grab_mid(mid, _feather, thread_num, 0); - scan_staus_count_inc(_mid); - - int virtual_table_id=0; - p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id); - if(p_table==NULL) - { - _feather->scan_err_cnt++; - return -1; - } - my_scanner=_feather->scanner; - if(my_scanner==NULL) - { - return 0; - } - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); - if(table_rt->origin_rule_num>0) // Even no rule in table, we still need to search for NOT compile. - { - if(p_table->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) - { - _feather->scan_err_cnt++; - return -1; - } - - intval_scan_data.rule_type=RULETYPE_INT; - intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0); - intval_scan_data.int_data=intval; - - alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - - region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; - - INC_SCANNER_REF(my_scanner,thread_num); - region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM); - if(region_ret<0) - { - DEC_SCANNER_REF(my_scanner, thread_num); - _feather->scan_err_cnt++; - return -1; - } - } - - if(region_ret>0 || scan_status_should_compile_NOT(_mid)) - { - struct scan_region_hit_wraper region_hit_wraper; - if(region_ret>0) - { - alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); - } - _mid=grab_mid(mid, _feather, thread_num, 1); - if(region_ret>0&&p_table->table_type==TABLE_TYPE_INTERVAL_PLUS) - { - district_id=_mid->district_id; - } - - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, district_id, - _mid->is_last_scan, virtual_table_id, _mid->scan_cnt); - - if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid)) - { - compile_ret=region_compile(_feather,_mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - } - } - - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num); - } - DEC_SCANNER_REF(my_scanner,thread_num); - - if(compile_ret==0&®ion_ret>0) - { - return -2; - } - return compile_ret; - -} - -int Maat_similar_scan_string(Maat_feather_t feather, int table_id, - const char* data, int data_len, - struct Maat_rule_t*result, int rule_num, - scan_status_t* mid, int thread_num) -{ - int hit_region_cnt=0,compile_ret=0; - struct _OUTER_scan_status_t* _mid=NULL; - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner* my_scanner=NULL; - Maat_table_schema* p_table=NULL; - struct timespec start,end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - _mid=grab_mid(mid, _feather, thread_num, 0); - scan_staus_count_inc(_mid); - - int virtual_table_id=0; - p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); - if(p_table==NULL) - { - _feather->scan_err_cnt++; - return -1; - } - if(p_table->table_type!=TABLE_TYPE_SIMILARITY) - { - return -1; - } - - my_scanner=_feather->scanner; - if(my_scanner==NULL) - { - return 0; - } - GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; - - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); - - GIE_handle_t* gie_handle=table_rt->similar.gie_handle; - if(gie_handle==NULL) - { - return 0; - } - INC_SCANNER_REF(my_scanner,thread_num); - alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - - hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM); - if(hit_region_cnt<0) - { - DEC_SCANNER_REF(my_scanner, thread_num); - _feather->scan_err_cnt++; - return -1; - } - else if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) - { - if(hit_region_cnt>0) - { - alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); - } - _mid=grab_mid(mid, _feather, thread_num, 1); - struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, - _mid->is_last_scan, virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(_feather,_mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - } - - DEC_SCANNER_REF(my_scanner,thread_num); - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt,0,&start, &end,thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt,0,NULL, NULL,thread_num); - } - if(compile_ret==0&&hit_region_cnt>0) - { - return -2; - } - return compile_ret; - -} -static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id) -{ - memset(scan_data, 0, sizeof(scan_data_t)); - scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0); - switch(addr->addrtype) - { - case ADDR_TYPE_IPV4: - scan_data->rule_type=RULETYPE_IPv4; - scan_data->ipv4_data.proto=proto; - switch(child_type) - { - case COMPONENT_TABLE_TYPE_SOURCE_IP: - scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); - scan_data->ipv4_data.sport=ntohs(addr->v4->source); - break; - case COMPONENT_TABLE_TYPE_DESTINATION_IP: - scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr); - scan_data->ipv4_data.sport=ntohs(addr->v4->dest); - break; - case COMPONENT_TABLE_TYPE_SESSION: - scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); - scan_data->ipv4_data.sport=ntohs(addr->v4->source); - scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr); - scan_data->ipv4_data.dport=ntohs(addr->v4->dest); - break; - default: - assert(0); - return -1; - } - break; - case ADDR_TYPE_IPV6: - scan_data->rule_type=RULETYPE_IPv6; - scan_data->ipv6_data.proto=proto; - switch(child_type) - { - case COMPONENT_TABLE_TYPE_SOURCE_IP: - memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); - ipv6_ntoh(scan_data->ipv6_data.saddr); - scan_data->ipv6_data.sport=ntohs(addr->v6->source); - break; - case COMPONENT_TABLE_TYPE_DESTINATION_IP: - memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr)); - ipv6_ntoh(scan_data->ipv6_data.saddr); - scan_data->ipv6_data.sport=ntohs(addr->v6->dest); - break; - case COMPONENT_TABLE_TYPE_SESSION: - memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); - ipv6_ntoh(scan_data->ipv6_data.saddr); - scan_data->ipv6_data.sport=ntohs(addr->v6->source); - - memcpy(scan_data->ipv6_data.daddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.daddr)); - ipv6_ntoh(scan_data->ipv6_data.daddr); - scan_data->ipv6_data.dport=ntohs(addr->v6->dest); - break; - default: - assert(0); - return -1; - break; - } - break; - default: - return -1; - } - return 0; -} -static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type, - scan_result_t *region_result, unsigned int result_num, - int* virtual_table_id, - rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr, - int thread_num) -{ - int child_table_id=0; - - if(child_type==COMPONENT_TABLE_TYPE_NONE) - { - child_table_id=parent_table_id; - child_type=COMPONENT_TABLE_TYPE_SESSION; - } - else - { - child_table_id=Maat_table_manager_get_child_id(table_mgr, parent_table_id, child_type); - } - if(child_table_id<0) - { - return 0; - } - - Maat_table_schema* real_table=Maat_table_manager_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); - if(real_table==NULL) - { - return 0; - } - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(table_rt_mgr, real_table->table_id); - if(table_rt->origin_rule_num==0) - { - return 0; - } - if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) - { - return 0; - } - if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) - { - return 0; - } - scan_data_t scan_data; - int ret=ip_scan_data_set(&scan_data, addr, proto, child_type, real_table->table_id); - if(ret<0) - { - return -1; - } - int region_ret=rulescan_search(ip_scanner, thread_num, &scan_data, region_result, result_num); - if(region_ret>0) - { - alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); - } - return region_ret; -} - -int Maat_scan_proto_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr,unsigned short int proto - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num) - -{ - int region_ret=0, compile_ret=0; - struct _OUTER_scan_status_t* _mid=NULL; - scan_result_t *region_result=NULL; - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner* my_scanner=NULL; - struct timespec start,end; - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - _mid=grab_mid(mid, _feather, thread_num, 0); - scan_staus_count_inc(_mid); - int virtual_table_id=0; - enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; - table_type=Maat_table_manager_get_type_by_id(_feather->table_mgr, table_id); - if(table_type==TABLE_TYPE_INVALID) - { - _feather->scan_err_cnt++; - return -1; - } - - my_scanner=_feather->scanner; - if(my_scanner==NULL) - { - return 0; - } - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, table_id); - - region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; - int region_hit_cnt=0; - int region_rslt_virtual_table_id[MAX_SCANNER_HIT_NUM]; - alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - INC_SCANNER_REF(my_scanner, thread_num); - - if(table_type==TABLE_TYPE_COMPOSITION) - { - enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION}; - for(int i=0; i<3; i++) - { - region_ret=IP_composition_scan(addr, proto, table_id, childs[i], - region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id, - my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); - if(region_ret<0) - { - _feather->scan_err_cnt++; - } - else if(region_ret>0) - { - for(int j=0; jregion, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); - if(region_ret<0) - { - _feather->scan_err_cnt++; - } - else if(region_ret>0) - { - region_hit_cnt+=region_ret; - } - - } - - struct scan_region_hit_wraper region_hit_wraper; - if(region_hit_cnt>0 || scan_status_should_compile_NOT(_mid) ) - { - _mid=grab_mid(mid, _feather, thread_num, 1); - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_hit_cnt, -1, - _mid->is_last_scan, virtual_table_id, _mid->scan_cnt); - if(table_type==TABLE_TYPE_COMPOSITION) - { - region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id; - } - compile_ret=region_compile(_feather,_mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - } - DEC_SCANNER_REF(my_scanner,thread_num); - if(_feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num); - } - if(compile_ret==0&®ion_hit_cnt>0) - { - return -2; - } - return compile_ret; -} -int Maat_scan_addr(Maat_feather_t feather,int table_id - ,struct ipaddr* addr - ,struct Maat_rule_t*result,int rule_num - ,scan_status_t *mid,int thread_num) -{ - int compile_ret=0; - compile_ret=Maat_scan_proto_addr(feather,table_id, - addr, 0, - result, rule_num, - mid,thread_num); - return compile_ret; -} -stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner* scanner=NULL; - - struct Maat_table_schema *p_table=NULL; - int virtual_table_id=0; - assert(thread_num<_feather->scan_thread_num); - p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); - if(p_table==NULL) - { - _feather->scan_err_cnt++; - return NULL; - } - - struct string_table_schema* expr_desc=&(p_table->expr); - struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); - scanner=_feather->scanner; - sp->feather=_feather; - sp->p_real_table=p_table; - sp->virtual_table_id=virtual_table_id; - sp->last_full_version=_feather->last_full_version; - sp->ref_scanner=_feather->scanner; - sp->process_offset=0; - sp->rs_stream_para=NULL; - - if(scanner==NULL) - { - return sp; - } - - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); - INC_SCANNER_REF(scanner, thread_num); - alignment_int64_array_add(table_rt->stream_num, thread_num, 1); - - if(table_rt->origin_rule_num==0) - { - return sp; - } - - sp->thread_num=thread_num; - sp->max_cross_size=expr_desc->cross_cache_size; - sp->caching_size=0; - sp->scan_buff=NULL; - sp->last_cache=NULL; - if(expr_desc->do_charset_merge==1) - { - sp->do_merge=1; - } - if(table_rt->expr.expr_rule_cnt>0) - { - sp->do_expr=1; - } - if(table_rt->expr.regex_rule_cnt>0) - { - sp->do_regex=1; - } - sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); - return sp; -} - -int Maat_stream_scan_string_detail(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid) -{ - struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner* scanner=sp->feather->scanner; - - int sub_type=0; - int region_ret=0,hit_region_cnt=0,compile_ret=0; - int district_id=-1; - struct _OUTER_scan_status_t* _mid=NULL; - scan_result_t *region_result; - scan_data_t region_scan_data; - struct timespec start,end; - if(sp->feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC, &start); - } - _mid=grab_mid(mid, sp->feather, sp->thread_num, 0); - scan_staus_count_inc(_mid); - if(data==NULL||data_len<=0||scanner==NULL) - { - return 0; - } - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); - - - if(sp->last_full_version!=sp->feather->last_full_version||sp->ref_scanner!=sp->feather->scanner) - { - return 0; - } - //table rule num is already tested in Maat_stream_scan_string_start - - - if(sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) - { - sp->feather->scan_err_cnt++; - return -1; - } - alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); - region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; - *detail_ret=0; - if(sp->do_merge==1) - { - sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE, 0); - } - else - { - sub_type=make_sub_type(sp->p_real_table->table_id, charset, 0); - } - if(sp->max_cross_size>0&&sp->caching_size>0) - { - if(sp->scan_buff!=NULL) - { - free(sp->scan_buff); - sp->scan_buff=NULL; - } - sp->scan_buff=ALLOC(char, sp->caching_size+data_len); - memcpy(sp->scan_buff,sp->last_cache,sp->caching_size); - memcpy(sp->scan_buff+sp->caching_size,data,data_len); - region_scan_data.text_data.text=sp->scan_buff; - region_scan_data.text_data.tlen=sp->caching_size+data_len; - } - else - { - region_scan_data.text_data.text=data; - region_scan_data.text_data.tlen=data_len; - } - region_scan_data.text_data.toffset=(int)MIN(0xffffffff/2, sp->process_offset);//longger then int - if(sp->last_cache==NULL&&sp->max_cross_size>0) - { - assert(sp->caching_size==0); - sp->last_cache=ALLOC(char, sp->max_cross_size); - } - if(sp->max_cross_size>0) - { - sp->caching_size=detain_last_data(sp->last_cache,sp->max_cross_size,sp->caching_size,data,data_len,&(sp->process_offset)); - } - else - { - sp->process_offset+=data_len; - } - if(sp->do_expr==1) - { - region_scan_data.rule_type=RULETYPE_STR; - region_scan_data.sub_type=sub_type; - region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result, MAX_SCANNER_HIT_NUM); - if(region_ret<0) - { - sp->feather->scan_err_cnt++; - return -1; - } - else if(region_ret>0) - { - hit_region_cnt+=region_ret; - } - } - if(sp->do_regex==1) - { - region_scan_data.rule_type=RULETYPE_REG; - region_scan_data.sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE,0); - region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); - if(region_ret<0) - { - sp->feather->scan_err_cnt++; - return -1; - } - else if(region_ret>0) - { - hit_region_cnt+=region_ret; - } - } - if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS) - { - district_id=_mid->district_id; - } - if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) - { - if(hit_region_cnt>0) - { - alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); - } - _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); - struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, district_id, - _mid->is_last_scan, sp->virtual_table_id, _mid->scan_cnt); - - if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid)) - { - compile_ret=region_compile(sp->feather, _mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - sp->thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0) - { - if(sp->scan_buff!=NULL) - { - *detail_ret=fill_region_hit_detail(sp->scan_buff, _mid->compile_mid, - region_result, hit_region_cnt, - result, compile_ret, - _mid->scan_cnt, - hit_detail, detail_num, scanner); - } - else - { - *detail_ret=fill_region_hit_detail(data, _mid->compile_mid, - region_result, hit_region_cnt, - result, compile_ret, - _mid->scan_cnt, - hit_detail, detail_num, scanner); - } - } - } - } - if(*detail_ret==0) - { - free(sp->scan_buff); - sp->scan_buff=NULL; - } - if(sp->feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt,data_len,&start, &end,sp->thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt,data_len,NULL, NULL,sp->thread_num); - } - if(compile_ret==0&&hit_region_cnt>0) - { - return -2; - } - return compile_ret; -} -int Maat_stream_scan_string(stream_para_t* stream_para - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid) - -{ - - int compile_ret=0; - int detail_ret=0; - compile_ret=Maat_stream_scan_string_detail(stream_para, charset,data,data_len, - result,rule_num, NULL, 0,&detail_ret,mid); - return compile_ret; -} -void Maat_stream_scan_string_end(stream_para_t* stream_para) -{ - struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime* table_rt=NULL; - if(scanner!=NULL) - { - table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); - alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); - - } - if(sp->rs_stream_para!=NULL) - { - if(scanner!=NULL&&sp->last_full_version==sp->feather->last_full_version&&sp->ref_scanner==sp->feather->scanner) - { - - DEC_SCANNER_REF(scanner, sp->thread_num); - rulescan_endstream(sp->rs_stream_para); - } - else - { - rulescan_endstream_simple(sp->rs_stream_para); - sp->feather->zombie_rs_stream--; - } - sp->rs_stream_para=NULL; - } - if(sp->last_cache!=NULL) - { - free(sp->last_cache); - sp->last_cache=NULL; - sp->caching_size=0; - } - if(sp->scan_buff!=NULL) - { - free(sp->scan_buff); - sp->scan_buff=NULL; - } - sp->ref_scanner=NULL; - sp->feather=NULL; - free(sp); - *stream_para=NULL; - return; -} - -stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_scanner* scanner=NULL; - sfh_instance_t * tmp_fuzzy_handle=NULL; - struct Maat_table_schema *p_table=NULL; - int virtual_table_id=0; - p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); - if(p_table==NULL) - { - _feather->scan_err_cnt++; - return NULL; - } - struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); - scanner=_feather->scanner; - sp->feather=_feather; - sp->ref_scanner=_feather->scanner; - sp->p_real_table=p_table; - sp->virtual_table_id=virtual_table_id; - sp->last_full_version=_feather->last_full_version; - sp->process_offset=0; - INC_SCANNER_REF(scanner, thread_num); - if(scanner==NULL) - { - return sp; - } - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id); - tmp_fuzzy_handle=SFH_instance(total_len); - if(tmp_fuzzy_handle==NULL) - { - _feather->scan_err_cnt++; - return NULL; - } - - sp->thread_num=thread_num; - sp->total_len=total_len; - sp->fuzzy_hash_handle=tmp_fuzzy_handle; - pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - alignment_int64_array_add(table_rt->stream_num,thread_num,1); - return sp; -} - -inline int REACH_QUERY_THRESH(unsigned long long total_len,unsigned long long acc_len,unsigned char* query_point,int point_size) -{ - const unsigned long long QUERY_MIN_RATE=(3); //30% -// const unsigned long long QUERY_MIN_LEN=(1024*1024*4); -//do query every 10 percent since 30%, e.g. 0.3/0.4/0.5/.../1.0 - unsigned long long rate=(acc_len*10)/total_len; -// if(acc_len>QUERY_MIN_LEN) -// { -// return 1; -// } - //TODO use a more thorough strategy. - if(total_len==0) - { - return 1; - } - if(rate>(unsigned long long)(point_size+QUERY_MIN_RATE)) - { - return 0; - } - if(rate>=QUERY_MIN_RATE&&query_point[rate-QUERY_MIN_RATE]==0) - { - query_point[rate-QUERY_MIN_RATE]=1; - return 1; - } - return 0; -} -int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int data_len, unsigned long long offset, struct Maat_rule_t * result, int rule_num, scan_status_t * mid) -{ - struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - int do_query=0; - int hit_region_cnt=0,compile_ret=0; - struct _OUTER_scan_status_t* _mid=NULL; - - struct timespec start, end; - if(sp->feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&start); - } - _mid=grab_mid(mid, sp->feather, sp->thread_num, 0); - scan_staus_count_inc(_mid); - - if(data==NULL||data_len<=0) - { - return 0; - } - if(sp->feather->scanner==NULL) - { - return 0; - } - GIE_result_t* region_result=sp->feather->scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; - - struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); - GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; - unsigned long long digest_len=0; - char* digest_buff=NULL; - - alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); - pthread_mutex_lock(&(sp->fuzzy_mutex)); - sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); - pthread_mutex_unlock(&(sp->fuzzy_mutex)); - do_query=REACH_QUERY_THRESH(sp->total_len, sp->process_offset, sp->query_point,8); - if(do_query==0) - { - goto fast_out; - } - pthread_mutex_lock(&(sp->fuzzy_mutex)); - digest_len=SFH_status(sp->fuzzy_hash_handle, HASH_LENGTH); - pthread_mutex_unlock(&(sp->fuzzy_mutex)); - if(digest_len==0) - { - goto fast_out; - } - digest_buff=ALLOC(char, digest_len); - pthread_mutex_lock(&(sp->fuzzy_mutex)); - SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); - pthread_mutex_unlock(&(sp->fuzzy_mutex)); - - if(GIE_handle!=NULL) - { - hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM); - } - - free(digest_buff); - digest_buff=NULL; - if(hit_region_cnt<0)//error occurs - { - sp->feather->scan_err_cnt++; - compile_ret=-1; - goto fast_out; - } - if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid)) - { - if(hit_region_cnt>0) - { - alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); - } - _mid=grab_mid(mid,sp->feather, sp->thread_num,1); - struct scan_region_hit_wraper region_hit_wraper; - scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, - _mid->is_last_scan, sp->virtual_table_id, _mid->scan_cnt); - compile_ret=region_compile(sp->feather, _mid->compile_mid, - ®ion_hit_wraper, - result, rule_num, - sp->thread_num); - assert(_mid->is_last_scan<2); - if(_mid->is_last_scan==1) - { - _mid->is_last_scan=2; - } - } -fast_out: - if(sp->feather->perf_on==1) - { - clock_gettime(CLOCK_MONOTONIC,&end); - Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, sp->thread_num); - } - else - { - Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, sp->thread_num); - } - if(compile_ret==0&&hit_region_cnt>0) - { - return -2; - } - return compile_ret; -} -void Maat_stream_scan_digest_end(stream_para_t* stream_para) -{ - struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); - struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); - alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); - if(scanner!=NULL) - { - if(sp->last_full_version==sp->feather->last_full_version) - { - DEC_SCANNER_REF(scanner, sp->thread_num); - } - } - SFH_release(sp->fuzzy_hash_handle); - pthread_mutex_destroy(&(sp->fuzzy_mutex)); - assert(sp->last_cache==NULL); - assert(sp->scan_buff==NULL); - sp->ref_scanner=NULL; - sp->feather=NULL; - free(sp); - *stream_para=NULL; - - return; -} -int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size) -{ - int ret=0; - struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_rule* compile=NULL; - switch(type) - { - case MAAT_RULE_SERV_DEFINE: - compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_read_user_data(_feather->scanner->hier, rule->config_id); - if(compile) - { - ret=MIN(size, compile->head.serv_def_len); - memcpy(value, compile->service_defined, ret); - } - else - { - ret=0; - } - break; - default: - ret=-1; - } - return ret; -} - -int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _OUTER_scan_status_t* _mid=NULL; - int map_ret=-1; - if(_feather->scanner==NULL) - { - return 0; - } - _mid=grab_mid(mid, _feather, -1, 0); - switch(type) - { - case MAAT_SET_SCAN_DISTRICT: - if(value==NULL||size<=0) - { - _feather->scan_err_cnt++; - return -1; - } - map_ret=maat_kv_read_unNull(_feather->scanner->district_map, (const char*)value, size, &(_mid->district_id)); - if(map_ret<0) - { - //May be the district is not effected yet. - _mid->district_id=DISTRICT_UNKNOWN; - } - _mid->is_set_district=1; - break; - case MAAT_SET_SCAN_LAST_REGION: - assert(_mid->is_last_scan==0); - _mid->is_last_scan=1; - break; - default: - _feather->scan_err_cnt++; - return -1; - break; - } - return 0; -} -int Maat_get_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, void* value, int size) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct _OUTER_scan_status_t* _mid=NULL; - struct Maat_hit_path_t* paths; - - size_t n_read=0; - int ret=0; - _mid=grab_mid(mid, _feather, 0, 0); - - if(_mid->compile_mid==NULL||_feather->scanner==NULL) - { - return 0; - } - switch(type) - { - case MAAT_GET_SCAN_HIT_PATH: - if(value==NULL||size<=0||size%sizeof(struct Maat_hit_path_t)!=0) - { - return -1; - } - paths=(struct Maat_hit_path_t*)value; - n_read=size/sizeof(struct Maat_hit_path_t); - n_read=Maat_hierarchy_get_hit_paths(_feather->scanner->hier , _mid->compile_mid, paths, n_read); - ret=n_read; - break; - default: - return -1; - } - return ret; -} - -int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int * int_val=(int *)value; - long long* longlong_val=(long long*)value; - switch(type) - { - case MAAT_STATE_VERSION: - if(size!=sizeof(long long)) - { - return -1; - } - if(_feather->new_version!=-1) - { - *longlong_val=_feather->new_version; - } - else - { - *longlong_val=_feather->maat_version; - } - break; - case MAAT_STATE_LAST_UPDATING_TABLE: - *int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr); - break; - case MAAT_STATE_IN_UPDATING: - if(size!=sizeof(int)) - { - return -1; - } - if(0==pthread_mutex_trylock(&(_feather->background_update_mutex))) - { - *int_val=0; - pthread_mutex_unlock(&(_feather->background_update_mutex)); - } - else - { - *int_val=1; - } - default: - return -1; - break; - } - return 0; -} - -void Maat_clean_status(scan_status_t* mid) -{ - struct _OUTER_scan_status_t* _mid=NULL; - if(*mid==NULL) - { - return; - } - _mid=(struct _OUTER_scan_status_t*)(*mid); - if(_mid->thread_num>=0) - { - alignment_int64_array_add(_mid->feather->outer_mid_cnt, _mid->thread_num, -1); - } - if(_mid->compile_mid!=NULL) - { - Maat_hierarchy_compile_mid_free(_mid->compile_mid); - _mid->compile_mid=NULL; - alignment_int64_array_add(_mid->feather->compile_mid_cnt, _mid->thread_num, -1); - } - _mid->feather=NULL; - free(_mid); - *mid=NULL; - return; -} - -int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len) -{ - return get_column_pos(line, Nth_column, column_offset, column_len); -} - diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp deleted file mode 100644 index 50ec952..0000000 --- a/src/entry/Maat_command.cpp +++ /dev/null @@ -1,2482 +0,0 @@ -#include "Maat_command.h" -#include "Maat_rule.h" -#include "Maat_rule_internal.h" -#include "Maat_utils.h" -#include "config_monitor.h" -#include "map_str2int.h" -#include "hiredis.h" -#include -#include -#include -#include -#include - -#define maat_redis_monitor (module_name_str("MAAT_REDIS_MONITOR")) -#define maat_command (module_name_str("MAAT_COMMAND")) -const time_t MAAT_REDIS_RECONNECT_INTERVAL=5; -const char* mr_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"}; -const char* mr_status_sset="MAAT_UPDATE_STATUS"; -const char* mr_expire_sset="MAAT_EXPIRE_TIMER"; -const char* mr_label_sset="MAAT_LABEL_INDEX"; -const char* mr_version_sset="MAAT_VERSION_TIMER"; -const char* mr_expire_lock="EXPIRE_OP_LOCK"; -const long mr_expire_lock_timeout=300*1000; -const static int MAAT_REDIS_SYNC_TIME=30*60; -const char* mr_op_str[]={"DEL","ADD","RENEW_TIMEOUT"}; -const char* foreign_source_prefix="redis://"; -const char* foreign_key_prefix="__FILE_"; - - - -int _wrap_redisGetReply(redisContext *c, redisReply **reply) -{ - return redisGetReply(c, (void **)reply); -} -redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...) -{ - va_list ap; - void *reply = NULL; - int ret=REDIS_ERR, retry=0; - while(reply==NULL&&retry<2&&ret!=REDIS_OK) - { - va_start(ap,format); - reply = redisvCommand(c,format,ap); - va_end(ap); - if(reply==NULL) - { - ret=redisReconnect(c); - retry++; - } - } - return (redisReply *)reply; -} -redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger) -{ - struct timeval connect_timeout; - connect_timeout.tv_sec=0; - connect_timeout.tv_usec=100*1000; // 100 ms - redisReply* reply=NULL; - - redisContext * ctx; - ctx=redisConnectWithTimeout(redis_ip, redis_port,connect_timeout); - if(ctx==NULL||ctx->err) - { - if(logger==NULL) - { - printf("Unable to connect redis server %s:%d db%d, error: %s\n", - redis_ip, redis_port, redis_db, ctx==NULL ? "Unknown" : ctx->errstr); - - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Unable to connect redis server %s:%d db%d, error: %s", - redis_ip, redis_port, redis_db, ctx==NULL ? "Unknown" : ctx->errstr); - } - if(ctx!=NULL) redisFree(ctx); - return NULL; - } - redisEnableKeepAlive(ctx); - reply=_wrap_redisCommand(ctx, "select %d",redis_db); - freeReplyObject(reply); - reply=NULL; - - return ctx; - -} - -int connect_redis_for_write(struct source_redis_ctx* mr_ctx, void* logger) -{ - assert(mr_ctx->write_ctx==NULL); - mr_ctx->write_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger); - if(mr_ctx->write_ctx==NULL) - { - return -1; - } - else - { - return 0; - } -} -redisContext* get_redis_ctx_for_write(struct _Maat_feather_t * feather) -{ - int ret=0; - if(feather->mr_ctx.write_ctx==NULL) - { - ret=connect_redis_for_write(&(feather->mr_ctx), feather->logger); - if(ret!=0) - { - return NULL; - } - } - return feather->mr_ctx.write_ctx; -} -long long read_redis_integer(const redisReply* reply) -{ - switch(reply->type) - { - case REDIS_REPLY_INTEGER: - return reply->integer; - break; - case REDIS_REPLY_ARRAY: - assert(reply->element[0]->type==REDIS_REPLY_INTEGER); - return reply->element[0]->integer; - break; - case REDIS_REPLY_STRING: - return atoll(reply->str); - break; - default: - return -1; - break; - } - return 0; -} -long long redis_server_time(redisContext* ctx) -{ - long long server_time=0; - redisReply* data_reply=NULL; - data_reply=_wrap_redisCommand(ctx,"TIME"); - if(data_reply->type==REDIS_REPLY_ARRAY) - { - server_time=atoll(data_reply->element[0]->str); - freeReplyObject(data_reply); - data_reply=NULL; - } - return server_time; -} -enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) -{ - enum MAAT_TABLE_TYPE ret=TABLE_TYPE_IP; - switch(p->region_type) - { - case REGION_IP: - ret=TABLE_TYPE_IP; - break; - case REGION_EXPR: - if(p->expr_rule.district==NULL) - { - ret=TABLE_TYPE_EXPR; - } - else - { - ret=TABLE_TYPE_EXPR_PLUS; - } - break; - case REGION_INTERVAL: - if(p->interval_rule.district==NULL) - { - ret=TABLE_TYPE_INTERVAL; - } - else - { - ret=TABLE_TYPE_INTERVAL_PLUS; - } - break; - case REGION_DIGEST: - ret=TABLE_TYPE_DIGEST; - break; - case REGION_SIMILARITY: - ret=TABLE_TYPE_SIMILARITY; - break; - default: - assert(0); - } - return ret; -} -int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type, int valid_column_seq) -{ - size_t offset=0, len=0; - unsigned int column_seq=0, ret=0; - switch(type) - { - case TABLE_TYPE_EXPR: - column_seq=7; - break; - case TABLE_TYPE_IP: - column_seq=14; - break; - case TABLE_TYPE_IP_PLUS: - column_seq=18; - break; - case TABLE_TYPE_COMPILE: - column_seq=8; - break; - case TABLE_TYPE_PLUGIN: - case TABLE_TYPE_IP_PLUGIN: - case TABLE_TYPE_FQDN_PLUGIN: - case TABLE_TYPE_BOOL_PLUGIN: - if(valid_column_seq<0) - { - return -1; - } - column_seq=(unsigned int)valid_column_seq; - break; - case TABLE_TYPE_INTERVAL: - column_seq=5; - break; - case TABLE_TYPE_INTERVAL_PLUS: - column_seq=6; - break; - case TABLE_TYPE_DIGEST: - column_seq=6; - break; - case TABLE_TYPE_SIMILARITY: - column_seq=5; - break; - case TABLE_TYPE_EXPR_PLUS: - column_seq=8; - break; - case TABLE_TYPE_GROUP2COMPILE: - case TABLE_TYPE_GROUP2GROUP: - column_seq=3; - break; - default: - assert(0); - } - - ret=get_column_pos(line, column_seq, &offset, &len); - if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. - { - return -1; - } - return offset; -} -int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) -{ - int i=0; - i=get_valid_flag_offset(line, type,valid_column_seq); - if(i<0) - { - return -1; - } - line[i]='0'; - return 0; -} - -void serialize_group2group(enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g, char* buff, size_t sz) -{ - snprintf(buff, sz, "%d\t%d\t%d", g2g->group_id, - g2g->superior_group_id, - op); - return; -} -void serialize_group2compile(enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c, char* buff, size_t sz) -{ - snprintf(buff, sz, "%d\t%d\t%d\t%d\t%s\t%d", g2c->group_id, - g2c->compile_id, - op, - g2c->not_flag, - g2c->virtual_table_name?g2c->virtual_table_name:"null", - g2c->clause_index); - return; -} -void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int clause_num, enum MAAT_OPERATION op, char* buff, size_t sz) -{ - if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; - const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null"); - - snprintf(buff, sz, "%d\t%d\t%hhu\t%hhu\t%hhu\t0\t%s\t%d\t%d", - p_m_rule->config_id, - p_m_rule->service_id, - p_m_rule->action, - p_m_rule->do_blacklist, - p_m_rule->do_log, - service_define, - op, - clause_num); - return; -} -void serialize_region(const struct Maat_cmd_region* p, int group_id, char* buff, size_t sz) -{ - UNUSED size_t ret=0; - switch(p->region_type) - { - case REGION_IP: - ret=snprintf(buff, sz, "%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1", - p->region_id, - group_id, - p->ip_rule.addr_type, - p->ip_rule.src_ip, - p->ip_rule.mask_src_ip, - p->ip_rule.src_port, - p->ip_rule.mask_src_port, - p->ip_rule.dst_ip, - p->ip_rule.mask_dst_ip, - p->ip_rule.dst_port, - p->ip_rule.mask_dst_port, - p->ip_rule.protocol, - p->ip_rule.direction); - break; - case REGION_IP_PLUS: - ret=snprintf(buff, sz, "%d\t%d\t%d\t%s\t%s\t%s\t%s\t%hu\t%hu\t%s\t%s\t%s\t%s\t%hu\t%hu\t%d\t%d\t1", - p->region_id, - group_id, - p->ip_plus_rule.addr_type, - p->ip_plus_rule.saddr_format, - p->ip_plus_rule.src_ip1, - p->ip_plus_rule.src_ip2, - p->ip_plus_rule.sport_format, - p->ip_plus_rule.src_port1, - p->ip_plus_rule.src_port2, - p->ip_plus_rule.daddr_format, - p->ip_plus_rule.dst_ip1, - p->ip_plus_rule.dst_ip2, - p->ip_plus_rule.dport_format, - p->ip_plus_rule.dst_port1, - p->ip_plus_rule.dst_port2, - p->ip_plus_rule.protocol, - p->ip_plus_rule.direction); - break; - case REGION_EXPR: - if(p->expr_rule.district==NULL) - { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1", - p->region_id, - group_id, - p->expr_rule.keywords, - p->expr_rule.expr_type, - p->expr_rule.match_method, - p->expr_rule.hex_bin); - } - else //expr_plus - { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1", - p->region_id, - group_id, - p->expr_rule.district, - p->expr_rule.keywords, - p->expr_rule.expr_type, - p->expr_rule.match_method, - p->expr_rule.hex_bin); - } - break; - case REGION_INTERVAL: - ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1", - p->region_id, - group_id, - p->interval_rule.low_boundary, - p->interval_rule.up_boundary); - break; - case REGION_DIGEST: - ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1", - p->region_id, - group_id, - p->digest_rule.orgin_len, - p->digest_rule.digest_string, - p->digest_rule.confidence_degree); - break; - case REGION_SIMILARITY: - ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1", - p->region_id, - group_id, - p->similarity_rule.target, - p->similarity_rule.threshold); - break; - default: - assert(0); - } - assert(rettable_line!=NULL) - { - free(rule->table_line); - } - if(rule->n_foreign>0) - { - for(int i=0; in_foreign; i++) - { - free(rule->f_keys[i].filename); - free(rule->f_keys[i].key); - } - free(rule->f_keys); - } - memset(rule,0,sizeof(struct serial_rule_t)); - return; -} -void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op, unsigned long rule_id,int label_id,const char* table_name,const char* line, long long timeout) -{ - memset(rule, 0, sizeof(struct serial_rule_t)); - rule->op=op; - rule->rule_id=rule_id; - rule->label_id=label_id; - rule->timeout=timeout; - assert(strlen(table_name)table_name)); - strncpy(rule->table_name, table_name, sizeof(rule->table_name)); - if(line!=NULL) - { - rule->table_line=_maat_strdup(line); - } - return; -} -int get_inc_key_list(long long instance_version, long long target_version, redisContext *c, struct serial_rule_t** list,void* logger) -{ - redisReply* reply=NULL,*tmp_reply=NULL; - char op_str[4]; - int rule_num=0; - UNUSED int ret=0; - unsigned int i=0, j=0; - long long nearest_rule_version; - struct serial_rule_t *s_rule=NULL; - - //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. - //The elements are considered to be ordered from low to high scores(instance_version). - reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",mr_status_sset,instance_version,target_version); - - if(reply==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "GET %s failed with a NULL reply, error: %s.", mr_status_sset, c->errstr); - return -1; - } - assert(reply->type==REDIS_REPLY_ARRAY); - rule_num=reply->elements; - if(reply->elements==0) - { - freeReplyObject(reply); - reply=NULL; - return 0; - } - - tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s", mr_status_sset,reply->element[0]->str); - if(tmp_reply->type!=REDIS_REPLY_STRING) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "ZSCORE %s %s failed Version: %lld->%lld", mr_status_sset, reply->element[0]->str, instance_version, target_version); - freeReplyObject(tmp_reply); - tmp_reply=NULL; - freeReplyObject(reply); - reply=NULL; - return -1; - } - nearest_rule_version=read_redis_integer(tmp_reply); - freeReplyObject(tmp_reply); - tmp_reply=NULL; - if(nearest_rule_version<0) - { - return -1; - } - if(nearest_rule_version!=instance_version+1) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Noncontinuous VERSION Redis: %lld MAAT: %lld.", nearest_rule_version, instance_version); - } - s_rule=ALLOC(struct serial_rule_t, reply->elements); - for(i=0, j=0;ielements;i++) - { - assert(reply->element[i]->type==REDIS_REPLY_STRING); - ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%lu", op_str, s_rule[j].table_name, &(s_rule[j].rule_id)); - if(ret!=3||s_rule[i].rule_id<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key: %s", reply->element[i]->str); - continue; - } - if(strncmp(op_str,"ADD",strlen("ADD"))==0) - { - s_rule[j].op=MAAT_OP_ADD; - } - else if(strncmp(op_str,"DEL",strlen("DEL"))==0) - { - s_rule[j].op=MAAT_OP_DEL; - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key: %s", reply->element[i]->str); - continue; - } - j++; - } - rule_num=j; - *list=s_rule; - freeReplyObject(reply); - reply=NULL; - - return rule_num; -} -struct s_rule_array_t -{ - int cnt; - int size; - struct serial_rule_t* array; -}; -void save_serial_rule_cb(const uchar * key, uint size, void * data, void * user) -{ - struct s_rule_array_t* array=(struct s_rule_array_t*)user; - int i=array->cnt; - memcpy(&(array->array[i]),data,sizeof(struct serial_rule_t)); - array->array[i].op=MAAT_OP_ADD; - return; -} -int recovery_history_version(const struct serial_rule_t* current, int current_num, const struct serial_rule_t* changed, int changed_num, struct serial_rule_t** history_result) -{ - int i=0,ret=0; - unsigned int history_num=0; - int hash_slot_size=1; - MESA_htable_handle htable=NULL; - MESA_htable_create_args_t hargs; - struct s_rule_array_t tmp_array; - char hkey[256+20]; - int tmp=current_num+changed_num; - for(;tmp>0;tmp=tmp/2) - { - hash_slot_size*=2; - } - - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=0; - hargs.hash_slot_size = hash_slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; - hargs.recursive = 1; - hargs.data_free = NULL;//data is an reference, no need to free. - hargs.data_expire_with_condition = NULL; - htable=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(htable, 0); - - for(i=0;i0); - } - - for(i=changed_num-1;i>=0;i--) - { - snprintf(hkey,sizeof(hkey),"%ld,%s",changed[i].rule_id,changed[i].table_name); - if(changed[i].op==MAAT_OP_ADD)//newly added rule is need to delete from current, so that history version can be recovered. - { - ret=MESA_htable_del(htable, (uchar*)hkey, strlen(hkey), NULL); - } - else - { - ret=MESA_htable_add(htable, (uchar*)hkey, strlen(hkey),changed+i); - } - if(ret<0)//failed - { - goto error_out; - } - } - history_num=MESA_htable_get_elem_num(htable); - tmp_array.cnt=0; - tmp_array.size=history_num; - tmp_array.array=(struct serial_rule_t*)calloc(history_num,sizeof(struct serial_rule_t)); - MESA_htable_iterate(htable, save_serial_rule_cb, &tmp_array); - *history_result=tmp_array.array; - ret=history_num; -error_out: - MESA_htable_destroy(htable, NULL); - return ret; -} - -int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct Maat_table_manager* table_mgr, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off) -{ - redisReply* reply=NULL,*sub_reply=NULL; - long long redis_version=0,target_version=0; - int rule_num=0, changed_rule_num=0, table_id=0; - int ret=0; - unsigned int i=0,full_idx =0,append_cmd_cnt=0; - struct serial_rule_t *s_rule_array=NULL, *changed_rule_array=NULL, *history_rule_array=NULL; - - reply=(redisReply*)redisCommand(c, "GET MAAT_VERSION"); - if(reply!=NULL) - { - - if(reply->type==REDIS_REPLY_NIL||reply->type==REDIS_REPLY_ERROR) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, "GET MAAT_VERSION failed, maybe Redis is busy."); - freeReplyObject(reply); - reply=NULL; - return -1; - } - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "GET MAAT_VERSION failed with NULL reply, error: %s.", c->errstr); - return -1; - } - redis_version=read_redis_integer(reply); - if(redis_version<0) - { - if(reply->type==REDIS_REPLY_ERROR) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Redis Communication error: %s.", reply->str); - } - return -1; - } - freeReplyObject(reply); - reply=NULL; - if(redis_version==instance_version) - { - return 0; - } - - if(instance_version==0||desired_version!=0) - { - goto FULL_UPDATE; - } - if(redis_version Redis: %lld.", instance_version, redis_version); - goto FULL_UPDATE; - } - if(redis_version>instance_version&&cumulative_off==1) - { - target_version=instance_version; - } - else - { - target_version=redis_version-1; - } - do{ - target_version++; - rule_num=get_inc_key_list(instance_version, target_version, c, &s_rule_array,logger); - if(rule_num>0) - { - break; - } - else if(rule_num<0) - { - goto FULL_UPDATE; - } - else - { - //ret=0, nothing to do. - } - - }while(rule_num==0&&target_version<=redis_version&&cumulative_off==1); - if(rule_num==0) - { - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_redis_monitor, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", - mr_status_sset, instance_version, target_version-1, cumulative_off==1?"OFF":"ON"); - return 0; - } - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Inc Update from instance_version %lld to %lld (%d entries).", instance_version,target_version,rule_num); - *list=s_rule_array; - *update_type=CM_UPDATE_TYPE_INC; - *new_version=target_version; - return rule_num; - -FULL_UPDATE: - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Initiate full udpate from instance_version %d to %lld.", instance_version, desired_version==0?redis_version:desired_version); - append_cmd_cnt=0; - ret=redisAppendCommand(c, "MULTI"); - append_cmd_cnt++; - ret=redisAppendCommand(c, "GET MAAT_VERSION"); - append_cmd_cnt++; - ret=redisAppendCommand(c, "KEYS EFFECTIVE_RULE:*"); - append_cmd_cnt++; - //consume reply "OK" and "QUEUED". - for(i=0;ierrstr); - return -1; - } - if(reply->type!=REDIS_REPLY_ARRAY) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key List type %d", reply->type); - freeReplyObject(reply); - reply=NULL; - return -1; - } - *new_version=read_redis_integer(reply->element[0]); - sub_reply=reply->element[1]; - if(sub_reply->type!=REDIS_REPLY_ARRAY) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key List type %d", sub_reply->type); - freeReplyObject(reply); - reply=NULL; - return -1; - } - - s_rule_array=(struct serial_rule_t*)calloc(sub_reply->elements,sizeof(struct serial_rule_t)); - for(i=0, full_idx=0; ielements; i++) - { - if(sub_reply->element[i]->type!=REDIS_REPLY_STRING) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key Type: %d", sub_reply->element[i]->type); - continue; - } - ret=sscanf(sub_reply->element[i]->str,"%*[^:]:%[^,],%ld",s_rule_array[full_idx].table_name,&(s_rule_array[full_idx].rule_id)); - s_rule_array[full_idx].op=MAAT_OP_ADD; - if(ret!=2||s_rule_array[full_idx].rule_id<0||strlen(s_rule_array[full_idx].table_name)==0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Invalid Redis Key Format: %s", sub_reply->element[i]->str); - continue; - } - if(table_mgr) - { - table_id=Maat_table_manager_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name); - if(table_id<0)//Unrecognized table. - { - continue; - } - } - full_idx++; - } - rule_num=full_idx; - freeReplyObject(reply); - reply=NULL; - if(desired_version!=0) - { - changed_rule_num=get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger); - if(changed_rule_num<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Recover history version %lld faild where as redis version is %lld.", desired_version, redis_version); - } - else if(changed_rule_num==0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Nothing to recover from history version %lld to redis version is %lld.", desired_version, redis_version); - } - else - { - ret=recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); - if(ret>0) - { - free(s_rule_array); - s_rule_array=history_rule_array; - rule_num=ret; - *new_version=desired_version; - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Successfully recovered from history version %lld to redis version is %lld.", desired_version, redis_version); - } - } - free(changed_rule_array); - } - *list=s_rule_array; - *update_type=CM_UPDATE_TYPE_FULL; - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "Full update %d keys of version %lld.", rule_num, *new_version); - - return rule_num ; -} - -int _get_maat_redis_value(redisContext *c, struct serial_rule_t* rule_list, int rule_num, void* logger) -{ - int i=0,failed_cnt=0,idx=0; - UNUSED int ret=0; - int error_happened=0; - int *retry_ids=(int*)malloc(sizeof(int)*rule_num); - char redis_cmd[256]; - redisReply* reply=NULL; - for(i=0;itype==REDIS_REPLY_STRING) - { - rule_list[i].table_line=_maat_strdup(reply->str); - } - else - { - if(reply->type==REDIS_REPLY_NIL) - { - retry_ids[failed_cnt]=i; - failed_cnt++; - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Redis GET %s:%s,%d failed",mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, - rule_list[i].rule_id); - error_happened=1; - } - } - freeReplyObject(reply); - reply=NULL; - } - if(error_happened==1) - { - free(retry_ids); - return -1; - } - - for(i=0;itype==REDIS_REPLY_STRING) - { - rule_list[idx].table_line=_maat_strdup(reply->str); - } - else if(reply->type==REDIS_REPLY_ERROR)//Deal with Redis response: "Loading Redis is loading the database in memory" - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "redis command %s error, reply type=%d, error str=%s", redis_cmd, reply->type, reply->str); - } - else //Handle type "nil" - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "redis command %s failed, reply type=%d", redis_cmd, reply->type); - } - - freeReplyObject(reply); - reply=NULL; - - } - free(retry_ids); - return 0; -} -int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process) -{ - int max_redis_batch=4*1024,batch_cnt=0; - int success_cnt=0,ret=0; - int next_print=10; - while(success_cntnext_print) - { - printf(" >%d%%",next_print); - next_print+=10; - } - } - } - if(print_process==1) - { - printf(" >100%%\n"); - } - return 0; -} - -int mr_transaction_success(redisReply* data_reply) -{ - if(data_reply->type==REDIS_REPLY_NIL) - { - return 0; - } - else - { - return 1; - } -} - -int redlock_try_lock(redisContext *ctx, const char* lock_name, long long expire) -{ - redisReply* reply=NULL; - int ret=0; - reply=_wrap_redisCommand(ctx,"SET %s locked NX PX %lld", lock_name, expire); - if(reply->type==REDIS_REPLY_NIL) - { - ret=0; - } - else - { - ret=1; - } - freeReplyObject(reply); - reply=NULL; - - return ret; -} -void redlock_unlock(redisContext * ctx, const char * lock_name) -{ - redisReply* reply=NULL; - reply=_wrap_redisCommand(ctx,"DEL %s", lock_name); - freeReplyObject(reply); - reply=NULL; - -} -#define POSSIBLE_REDIS_REPLY_SIZE 2 -struct expected_reply -{ - int srule_seq; - int possible_reply_num; - redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE]; -}; -void expected_reply_add(struct expected_reply* expected, int srule_seq, int type, long long integer) -{ - int i=expected->possible_reply_num; - assert(isrule_seq=srule_seq; - expected->possible_replies[i].type=type; - expected->possible_replies[i].integer=integer; - expected->possible_reply_num++; -} -int mr_operation_success(redisReply* actual_reply, struct expected_reply* expected) -{ - int i=0; - if(expected->possible_replies[0].type!=actual_reply->type) - { - return 0; - } - for(i=0; i< expected->possible_reply_num; i++) - { - if(expected->possible_replies[i].type==REDIS_REPLY_INTEGER && - expected->possible_replies[i].type==actual_reply->type && - expected->possible_replies[i].integer==actual_reply->integer) - { - return 1; - } - if(expected->possible_replies[i].type==REDIS_REPLY_STATUS && - expected->possible_replies[i].type==actual_reply->type && - 0==strcasecmp(actual_reply->str, "OK")) - { - return 1; - } - } - return 0; - -} - -long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule_num,int *renew_allowed, long long *transaction_version) -{ - int ret=-1; - redisReply* data_reply=NULL; - if(renew_rule_num>0) - { - while(0==redlock_try_lock(ctx, mr_expire_lock, mr_expire_lock_timeout)) - { - usleep(1000); - } - *renew_allowed=1; - } - if(rule_num>renew_rule_num) - { - data_reply=_wrap_redisCommand(ctx, "INCRBY MAAT_PRE_VER 1"); - *transaction_version=read_redis_integer(data_reply); - freeReplyObject(data_reply); - data_reply=NULL; - if(*transaction_version<0) - { - return -1; - } - } - if(*renew_allowed==1||rule_num>renew_rule_num) - { - data_reply=_wrap_redisCommand(ctx,"MULTI"); - freeReplyObject(data_reply); - data_reply=NULL; - ret=0; - } - return ret; -} -//parameters: 4 keys: MAAT_VERSION MAAT_UPDATE_STATUS MAAT_VERSION_TIMER MAAT_TRANSACTION_xx, 1 args: SERVER_TIME -const char* lua_exec_done= -"local maat_version=redis.call(\'incrby\', KEYS[1], 1);" -"local transaction=redis.call(\'lrange\', KEYS[4], 0, -1);" -"for k,v in pairs(transaction) do" -" redis.call(\'zadd\', KEYS[2], maat_version, v);" -"end;" -"redis.call(\'del\', KEYS[4]);" -"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);" -"return maat_version;"; -redisReply* _exec_serial_rule_end(redisContext* ctx, const char* transaction_list, long long server_time, int renew_allowed, struct expected_reply* expect_reply, unsigned int *cnt) -{ - redisReply* data_reply=NULL; - if(renew_allowed==1) - { - redlock_unlock(ctx, mr_expire_lock); - expect_reply[*cnt].srule_seq=-1; - (*cnt)++; - } - if(strlen(transaction_list)>0) - { - data_reply=_wrap_redisCommand(ctx, "eval %s 4 MAAT_VERSION %s %s %s %lld", - lua_exec_done, - mr_status_sset, - mr_version_sset, - transaction_list, - server_time); - freeReplyObject(data_reply); - data_reply=NULL; - expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - } - data_reply=_wrap_redisCommand(ctx,"EXEC"); - return data_reply; -} - -void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply* expect_reply, unsigned int *cnt, int offset,int renew_allowed) -{ - redisReply* data_reply=NULL; - unsigned int append_cmd_cnt=0, i=0; - for(i=0;i0) - { - redisAppendCommand(ctx,"ZADD %s %lld %s,%lu", - mr_expire_sset, - s_rule[i].timeout, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - append_cmd_cnt++; - } - if(s_rule[i].label_id>0) - { - redisAppendCommand(ctx,"ZADD %s %d %s,%lu", - mr_label_sset, - s_rule[i].label_id, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0); - - (*cnt)++; - - append_cmd_cnt++; - } - break; - case MAAT_OP_DEL: - redisAppendCommand(ctx,"RENAME %s:%s,%lu %s:%s,%lu", - mr_key_prefix[MAAT_OP_ADD], - s_rule[i].table_name, - s_rule[i].rule_id, - mr_key_prefix[MAAT_OP_DEL], - s_rule[i].table_name, - s_rule[i].rule_id - ); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0); - (*cnt)++; - append_cmd_cnt++; - - redisAppendCommand(ctx,"EXPIRE %s:%s,%lu %d", - mr_key_prefix[MAAT_OP_DEL], - s_rule[i].table_name, - s_rule[i].rule_id, - MAAT_REDIS_SYNC_TIME); - expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); - (*cnt)++; - append_cmd_cnt++; - - //NX: Don't update already exisiting elements. Always add new elements. - redisAppendCommand(ctx,"RPUSH %s DEL,%s,%lu", - transaction_list, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - append_cmd_cnt++; - - // Try to remove from expiration sorted set, no matter wheather it exists or not. - redisAppendCommand(ctx,"ZREM %s %s,%lu", - mr_expire_sset, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - append_cmd_cnt++; - - // Try to remove from label sorted set, no matter wheather it exists or not. - redisAppendCommand(ctx,"ZREM %s %s,%lu", - mr_label_sset, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - append_cmd_cnt++; - break; - case MAAT_OP_RENEW_TIMEOUT: - if(renew_allowed!=1) - { - continue; - } - //s_rule[i].timeout>0 was checked by caller. - redisAppendCommand(ctx,"ZADD %s %lld %s,%lu", - mr_expire_sset, - s_rule[i].timeout, - s_rule[i].table_name, - s_rule[i].rule_id); - expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); - (*cnt)++; - append_cmd_cnt++; - - break; - default: - assert(0); - break; - } - } - for(i=0;i0) - { - snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version); - } - while(success_cntelements==multi_cmd_cnt); - for(i=0;ielement[i]; - //failed is acceptable - //or transaciton is success - //or continuation of last failed - if(expected_reply[i].srule_seq==-1||1==mr_operation_success(p, expected_reply+i)||last_failed==expected_reply[i].srule_seq) - { - continue; - } - rule_seq=expected_reply[i].srule_seq; - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_command, - "%s %s %d failed, rule id maybe conflict or not exist.", - mr_op_str[s_rule[rule_seq].op], - s_rule[rule_seq].table_name,s_rule[rule_seq].rule_id); - success_cnt--; - last_failed=rule_seq; - } - } - else - { - success_cnt=-1; - } - if(transaction_version>0) - { - transaction_finished_version=read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]); - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_command, - "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld ", - transaction_version, - transaction_finished_version); - } - - freeReplyObject(transaction_reply); - transaction_reply=NULL; - -error_out: - if(renew_num>0&&renew_allowed!=1) - { - for(i=0;i<(unsigned int)serial_rule_num;i++) - { - if(s_rule[i].op==MAAT_OP_RENEW_TIMEOUT) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_command - ,"%s %s %d is not allowed due to lock contention.",mr_op_str[MAAT_OP_RENEW_TIMEOUT] - , s_rule[i].table_name,s_rule[i].rule_id); - } - } - if(success_cnt>0) - { - success_cnt-=renew_num; - } - } - free(expected_reply); - return success_cnt; -} - - -void check_maat_expiration(redisContext *ctx, void *logger) -{ - unsigned int i=0,s_rule_num=0; - UNUSED int ret=0; - int success_cnt=0; - redisReply* data_reply=NULL; - struct serial_rule_t* s_rule=NULL; - long long server_time=0; - - server_time=redis_server_time(ctx); - if(!server_time) - { - return; - } - data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time); - if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0) - { - freeReplyObject(data_reply); - data_reply=NULL; - return; - } - s_rule_num=data_reply->elements; - s_rule=(struct serial_rule_t*)calloc(sizeof(struct serial_rule_t),s_rule_num); - for(i=0;ielement[i]->str,"%[^,],%ld",s_rule[i].table_name,&(s_rule[i].rule_id)); - assert(ret==2); - } - freeReplyObject(data_reply); - data_reply=NULL; - success_cnt=exec_serial_rule(ctx,s_rule, s_rule_num,server_time, logger); - - if(success_cnt==(int)s_rule_num) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor - ,"Succesfully expired %d rules in Redis.", s_rule_num); - } - else - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor - ,"Failed to expired %d of %d rules in Redis, try later.", s_rule_num-success_cnt,s_rule_num); - } - - free(s_rule); - return; -} -void cleanup_update_status(redisContext *ctx, void *logger) -{ - redisReply* reply=NULL,*sub_reply=NULL; - int append_cmd_cnt=0,i=0; - long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0; - - server_time=redis_server_time(ctx); - if(!server_time) - { - return; - } - reply=_wrap_redisCommand(ctx,"MULTI"); - freeReplyObject(reply); - reply=NULL; - redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME); - append_cmd_cnt++; - redisAppendCommand(ctx, "ZREMRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME); - append_cmd_cnt++; - //consume reply "OK" and "QUEUED". - for(i=0;itype!=REDIS_REPLY_ARRAY) - { - goto error_out; - } - sub_reply=reply->element[0]; - if(sub_reply->type!=REDIS_REPLY_ARRAY) - { - goto error_out; - } - version_num=sub_reply->elements; - if(version_num==0) - { - goto error_out; - } - version_lower_bound=read_redis_integer(sub_reply->element[0]); - version_upper_bound=read_redis_integer(sub_reply->element[sub_reply->elements-1]); - freeReplyObject(reply); - reply=NULL; - - //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally. - reply=_wrap_redisCommand(ctx,"ZREMRANGEBYSCORE %s %lld %lld",mr_status_sset,version_lower_bound,version_upper_bound); - entry_num=read_redis_integer(reply); - freeReplyObject(reply); - reply=NULL; - - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor - ,"Clean up update status from version %lld to %lld (%lld versions, %lld entries)." - ,version_lower_bound - ,version_upper_bound - ,version_num - ,entry_num); - return; - -error_out: - freeReplyObject(reply); - reply=NULL; - return; -} -const char* find_Nth_column(const char* line, int Nth, int* column_len) -{ - size_t i=0; - int j=0; - int start=0, end=0; - size_t line_len= strlen(line); - for(i=0;in_foreign; i++) - { - fn_size+=strlen(p->f_keys[i].filename); - } - - rewrite_line=(char*)calloc(sizeof(char), strlen(p->table_line)+fn_size); - pos_origin_line=p->table_line; - pos_rewrite_line=rewrite_line; - - for(i=0; in_foreign; i++) - { - origin_column=find_Nth_column(p->table_line, p->f_keys[i].column, &origin_column_size); - strncat(pos_rewrite_line, pos_origin_line, origin_column-pos_origin_line); - pos_rewrite_line+=origin_column-pos_origin_line; - pos_origin_line=origin_column+origin_column_size; - - strncat(pos_rewrite_line, p->f_keys[i].filename, strlen(p->f_keys[i].filename)); - pos_rewrite_line+=strlen(p->f_keys[i].filename); - } - strncat(pos_rewrite_line, pos_origin_line, strlen(p->table_line)-(pos_origin_line-p->table_line)); - - free(p->table_line); - p->table_line=rewrite_line; - return; -} -void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger) -{ - int i=0; - const char* p_foreign=NULL; - int foreign_key_size=0; - p_rule->f_keys=ALLOC(struct foreign_key, n_foreign); - for(i=0; itable_line, foreign_columns[i], &foreign_key_size); - if(p_foreign==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Get %s,%d foreign keys failed: No %dth column.", - p_rule->table_name, p_rule->rule_id, foreign_columns[i]); - continue; - } - if(0==strncasecmp(p_foreign, "null", strlen("null"))) - {//emtpy file - continue; - } - if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor, - "Get %s,%d foreign key failed: Invalid source prefix %s.", - p_rule->table_name, p_rule->rule_id, p_foreign); - continue; - } - p_rule->f_keys[p_rule->n_foreign].column=foreign_columns[i]; - foreign_key_size=foreign_key_size-strlen(foreign_source_prefix); - p_foreign+=strlen(foreign_source_prefix); - if(0!=strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, - "%s, %d foreign key prefix %s is not recommended.", - p_rule->table_name, p_rule->rule_id, p_foreign); - } - p_rule->f_keys[p_rule->n_foreign].key=ALLOC(char, foreign_key_size+1); - memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); - p_rule->f_keys[p_rule->n_foreign].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir); - p_rule->n_foreign++; - } - if(p_rule->n_foreign==0) - { - free(p_rule->f_keys); - p_rule->f_keys=NULL; - } - return; -} -int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) -{ - int i=0; - int rule_with_foreign_key=0; - struct Maat_table_schema* p_table=NULL; - struct plugin_table_schema* plugin_desc=NULL; - for(i=0; itable_mgr, rule_list[i].table_name); - if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN) - { - continue; - } - plugin_desc= &(p_table->plugin); - if(plugin_desc->n_foreign==0) - { - continue; - } - _get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger); - rule_with_foreign_key++; - } - return rule_with_foreign_key; -} -int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger) -{ - int i=0, j=0, foreign_key_size=0; - int rule_with_foreign_key=0; - const char* p_foreign=NULL; - - int n_foreign=0; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - for(i=0; i(int)strlen(foreign_source_prefix)&&0==strncmp(p_foreign,foreign_source_prefix, strlen(foreign_source_prefix))) - { - foreign_columns[n_foreign]=j; - n_foreign++; - } - j++; - }while(p_foreign!=NULL&&n_foreign0) - { - _get_foregin_keys(rule_list+i, foreign_columns, n_foreign,dir,logger); - rule_with_foreign_key++; - } - } - return rule_with_foreign_key; -} - -struct foreign_conts_track -{ - int rule_idx; - int foreign_idx; -}; -void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger) -{ - int i=0, j=0; - UNUSED int ret=0; - int key_num=0; - struct foreign_conts_track* track=ALLOC(struct foreign_conts_track, rule_num*MAX_FOREIGN_CLMN_NUM); - char redis_cmd[256]; - redisReply* reply=NULL; - struct serial_rule_t*p=NULL; - FILE* fp=NULL; - struct stat file_info; - - for(i=0;in_foreign==0) - { - continue; - } - if(p->op==MAAT_OP_DEL) - { - for(j=0; jn_foreign; j++) - { - if(rule_list[i].f_keys[j].filename==NULL) - { - continue; - } - ret=stat(p->f_keys[j].filename, &file_info); - if(ret==0) - { - continue; - } - snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); - ret=redisAppendCommand(ctx, redis_cmd); - track[key_num].rule_idx=i; - track[key_num].foreign_idx=j; - key_num++; - assert(ret==REDIS_OK); - } - } - } - for(i=0;itype!=REDIS_REPLY_STRING) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor - ,"Get %s,%d foreign key %s content failed." - ,rule_list[track[i].rule_idx].table_name - ,rule_list[track[i].rule_idx].rule_id - ,rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); - continue; - } - else - { - p=rule_list+track[i].rule_idx; - fp=fopen(p->f_keys[track[i].foreign_idx].filename, "w"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor - , "Write foreign content failed: fopen %s error." - , p->f_keys[track[i].foreign_idx]); - } - else - { - fwrite(reply->str, 1, reply->len, fp); - fclose(fp); - fp=NULL; - if(print_fn==1) - { - printf("Written foreign content %s\n",p->f_keys[track[i].foreign_idx].filename); - } - } - } - freeReplyObject(reply); - reply=NULL; - } - - free(track); - return; -} -void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger) -{ - int max_redis_batch=4*1024,batch_cnt=0; - int success_cnt=0; - while(success_cntlogger; - - if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write - { - //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. - if(1==redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout)) - { - check_maat_expiration(mr_ctx->read_ctx, logger); - cleanup_update_status(mr_ctx->read_ctx, logger); - redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); - } - } - if(mr_ctx->read_ctx==NULL||mr_ctx->read_ctx->err) - { - if(time(NULL)-mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL) - { - return; - } - mr_ctx->last_reconnect_time=time(NULL); - if(mr_ctx->read_ctx!=NULL) - { - redisFree(mr_ctx->read_ctx); - } - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Reconnecting..."); - mr_ctx->read_ctx=connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, feather->logger); - if(mr_ctx->read_ctx==NULL) - { - return; - } - else - { - version=0;//Trigger full update when reconnect to redis. - } - } - - rule_num=get_rm_key_list(mr_ctx->read_ctx, version, feather->load_version_from, &new_version, feather->table_mgr, &rule_list, &update_type, logger, feather->cumulative_update_off); - if(rule_num<0)//redis communication error - { - redisFree(mr_ctx->read_ctx); - mr_ctx->read_ctx=NULL; - return; - } - feather->load_version_from=0;//only valid for one time. - if(rule_num==0&&update_type==CM_UPDATE_TYPE_INC)//error or nothing changed - { - return; - } - if(rule_num>0) - { - ret=get_maat_redis_value(mr_ctx->read_ctx, rule_list, rule_num, logger, 0); - if(ret<0)//redis communication error - { - redisFree(mr_ctx->read_ctx); - mr_ctx->read_ctx=NULL; - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Get Redis value failed, abandon update and close connection."); - goto clean_up; - } - for(i=0;iread_ctx, rule_list, rule_num, feather, feather->foreign_cont_dir, logger); - if(ret>0) - { - get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, logger); - } - } - start(new_version, update_type, u_para); - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Start %s update: %lld -> %lld (%d entries).", - update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); - for(i=0;itable_mgr, rule_list[i].table_name); - if(table_id<0)//Unrecognized table. - { - no_table_num++; - continue; - } - table_type=Maat_table_manager_get_type_by_id(feather->table_mgr, table_id); - if(rule_list[i].op==MAAT_OP_DEL) - { - - scan_type=Maat_table_get_scan_type(table_type); - table_schema=Maat_table_manager_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL); - valid_column=Maat_table_schema_get_valid_flag_column(table_schema); - ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor, "Invalidate line failed, invaid format %s .", - rule_list[i].table_line); - continue; - } - } - if(rule_list[i].n_foreign>0) - { - rewrite_table_line_with_foreign(rule_list+i); - } - update(rule_list[i].table_name,rule_list[i].table_line,u_para); - call_update_num++; - } - finish(u_para); - if(call_update_numtable_name!=NULL) - { - dst->table_name=_maat_strdup(src->table_name); - } - switch(dst->region_type) - { - case REGION_IP: - dst->ip_rule.src_ip=_maat_strdup(src->ip_rule.src_ip); - dst->ip_rule.mask_src_ip=_maat_strdup(src->ip_rule.mask_src_ip); - dst->ip_rule.dst_ip=_maat_strdup(src->ip_rule.dst_ip); - dst->ip_rule.mask_dst_ip=_maat_strdup(src->ip_rule.mask_dst_ip); - break; - case REGION_EXPR: - dst->expr_rule.keywords=_maat_strdup(src->expr_rule.keywords); - dst->expr_rule.district=_maat_strdup(src->expr_rule.district); - break; - case REGION_INTERVAL: - break; - case REGION_DIGEST: - dst->digest_rule.digest_string=_maat_strdup(src->digest_rule.digest_string); - break; - case REGION_SIMILARITY: - dst->similarity_rule.target=_maat_strdup(src->similarity_rule.target); - break; - default: - assert(0); - } - return; -} -void _maat_empty_region(struct Maat_region_t* p) -{ - free((char*)p->table_name); - p->table_name=NULL; - switch(p->region_type) - { - case REGION_IP: - free((char*)p->ip_rule.src_ip); - free((char*)p->ip_rule.mask_src_ip); - free((char*)p->ip_rule.dst_ip); - free((char*)p->ip_rule.mask_dst_ip); - break; - case REGION_EXPR: - free((char*)p->expr_rule.keywords); - free((char*)p->expr_rule.district); - break; - case REGION_INTERVAL: - break; - case REGION_DIGEST: - free((char*)p->digest_rule.digest_string); - break; - case REGION_SIMILARITY: - free((char*)p->similarity_rule.target); - break; - default: - assert(0); - } - memset(p,0,sizeof(struct Maat_region_t)); - return; - -} -int Maat_command_raw_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, size_t n_line ,enum MAAT_OPERATION op) -{ - size_t i=0; - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=0, table_id=0,success_cnt=0; - struct serial_rule_t *s_rule=NULL; - long long server_time=0,absolute_expire_time=0; - redisContext* write_ctx=get_redis_ctx_for_write(_feather); - if(write_ctx==NULL) - { - return -1; - } - server_time=redis_server_time(write_ctx); - if(!server_time) - { - return -1; - } - s_rule=ALLOC(struct serial_rule_t, n_line); - for(i=0;itable_mgr, line_rule[i]->table_name); - if(table_id<0) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, - "Command raw set line id %d failed: unknown table %s.", - line_rule[i]->rule_id, - line_rule[i]->table_name); - ret=-1; - goto error_out; - } - if(op==MAAT_OP_RENEW_TIMEOUT) - { - assert(line_rule[i]->expire_after>0); - } - if(line_rule[i]->expire_after>0) - { - absolute_expire_time=server_time+line_rule[i]->expire_after; - } - set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name, - line_rule[i]->table_line, absolute_expire_time); - } - success_cnt=exec_serial_rule(write_ctx,s_rule, n_line,server_time,_feather->logger); - if(success_cnt<0||(size_t)success_cnt!=n_line)//error - { - ret=-1; - goto error_out; - } - ret=success_cnt; - _feather->line_cmd_acc_num+=success_cnt; - -error_out: - for(i=0;itable_mgr, line_rule[i]->table_name); - if(table_id<0) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command - ,"Command set line id %d failed: unknown table %s." - , line_rule[i]->rule_id - , line_rule[i]->table_name); - ret=-1; - goto error_out; - } - p_table=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id); - if(!p_table) - { - ret=-1; - goto error_out; - } - int valid_flag_column=0; - - valid_flag_column=Maat_table_schema_get_valid_flag_column(p_table); - if(valid_flag_column<0) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, - "Command set line id %d failed: table %s is not a plugin or ip_plugin table.", - line_rule[i]->rule_id, - line_rule[i]->table_name); - ret=-1; - goto error_out; - - } - - if(op==MAAT_OP_ADD) - { - ret=get_valid_flag_offset(line_rule[i]->table_line, - p_table->table_type, - valid_flag_column); - if(ret<0|| - (op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1')) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, - "Command set line %s %d failed: illegal valid flag.", - line_rule[i]->table_name, line_rule[i]->rule_id); - ret=-1; - goto error_out; - } - } - if(op==MAAT_OP_RENEW_TIMEOUT) - { - assert(line_rule[i]->expire_after>0); - } - if(line_rule[i]->expire_after>0) - { - absolute_expire_time=server_time+line_rule[i]->expire_after; - } - if(plugin_desc && plugin_desc->n_foreign>0) - { - for(j=0;jn_foreign;j++) - { - p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size); - if(p_foreign==NULL) - { - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command, - "Command set line %s %d failed: No %dth column.", - line_rule[i]->table_name, line_rule[i]->rule_id, - plugin_desc->foreign_columns[j]); - ret=-1; - goto error_out; - } - if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_redis_monitor, - "Command set line %s %d failed: Source prefix %s is mandatory.", - line_rule[i]->table_name, line_rule[i]->rule_id, foreign_source_prefix); - ret=-1; - goto error_out; - } - } - - } - set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name, - line_rule[i]->table_line, absolute_expire_time); - } - success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger); - if(success_cnt<0||success_cnt!=line_num)//error - { - ret=-1; - goto error_out; - } - ret=success_cnt; - _feather->line_cmd_acc_num+=success_cnt; - -error_out: - for(i=0;imr_ctx.write_ctx; - if(ctx==NULL) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__); - return -1; - } - const char *arg_vec[3]; - size_t len_vec[3]; - arg_vec[0] = "SET"; - len_vec[0] = strlen("SET"); - - arg_vec[1] = key; - len_vec[1] = strlen(key); - - arg_vec[2] = value; - len_vec[2] = size; - - redisReply *reply=NULL; - if(0!=strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix))) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "Invalid File key, prefix %s is mandatory.", foreign_key_prefix); - return -1; - } - switch(op) - { - case MAAT_OP_ADD: - reply= (redisReply *)redisCommandArgv(ctx, sizeof(arg_vec) / sizeof(arg_vec[0]), arg_vec, len_vec); - break; - case MAAT_OP_DEL: - reply=_wrap_redisCommand(ctx, "EXPIRE %s %d", key, MAAT_REDIS_SYNC_TIME); - break; - default: - return -1; - break; - } - if(reply==NULL||reply->type==REDIS_REPLY_NIL||reply->type==REDIS_REPLY_ERROR) - { - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,"Set file failed, maybe Redis is busy."); - freeReplyObject(reply); - reply=NULL; - return -1; - } - freeReplyObject(reply); - reply=NULL; - return 1; -} - -long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - redisReply* data_reply=NULL; - long long result=0; - redisContext* write_ctx=get_redis_ctx_for_write(_feather); - - if(write_ctx==NULL) - { - return -1; - } - data_reply=_wrap_redisCommand(write_ctx, "INCRBY %s %d", key, increment); - if(data_reply->type==REDIS_REPLY_INTEGER) - { - result=data_reply->integer; - } - else - { - result=-1; - } - freeReplyObject(data_reply); - data_reply=NULL; - return result; -} -int Maat_command_get_new_group_id(Maat_feather_t feather) -{ - int group_id=0; - group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1); - return group_id; -} -int Maat_command_get_new_region_id(Maat_feather_t feather) -{ - int region_id=0; - region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1); - return region_id; -} - -void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size) -{ - int i=0; - struct Maat_cmd_key* p=*keys; - for(i=0; itable_name); - p->table_name=NULL; - p->rule_id=0; - } - free(*keys); - *keys=NULL; - return; -} - -int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - redisReply* data_reply=NULL; - char* tmp=NULL; - unsigned int i=0; - struct Maat_cmd_key* result=NULL; - int result_cnt=0; - redisContext* write_ctx=get_redis_ctx_for_write(_feather); - if(write_ctx==NULL) - { - return -1; - } - - data_reply=_wrap_redisCommand(write_ctx,"ZRANGEBYSCORE %s %d %d", - mr_label_sset, - label_id, - label_id); - result_cnt=data_reply->elements; - result=ALLOC(struct Maat_cmd_key, data_reply->elements); - for(i=0;ielements;i++) - { - result[i].table_name=_maat_strdup(data_reply->element[i]->str); - tmp=strchr(result[i].table_name, ','); - if(tmp!=NULL) - { - *tmp='\0'; - tmp++; - result[i].rule_id=atoi(tmp); - } - else// old version compatible - { - result[i].rule_id=atoi(result[i].table_name); - free(result[i].table_name); - result[i].table_name=NULL; - } - } - freeReplyObject(data_reply); - data_reply=NULL; - - *keys=result; - return result_cnt; -} - -int redis_flush_DB(redisContext* ctx, int db_index, void* logger) -{ - redisReply* data_reply=NULL; - long long maat_redis_version=0, dbsize=0; - int append_cmd_cnt=0, i=0,ret=0; - int redis_transaction_success=1; - - data_reply=_wrap_redisCommand(ctx, "WATCH MAAT_VERSION"); - freeReplyObject(data_reply); - data_reply=NULL; - data_reply=_wrap_redisCommand(ctx, "GET MAAT_VERSION"); - if(data_reply->type==REDIS_REPLY_NIL) - { - maat_redis_version=0; - } - else - { - maat_redis_version=read_redis_integer(data_reply); - maat_redis_version++; - freeReplyObject(data_reply); - data_reply=NULL; - } - data_reply=_wrap_redisCommand(ctx, "DBSIZE"); - dbsize=read_redis_integer(data_reply); - freeReplyObject(data_reply); - data_reply=NULL; - - data_reply=_wrap_redisCommand(ctx,"MULTI"); - freeReplyObject(data_reply); - data_reply=NULL; - - redisAppendCommand(ctx,"FLUSHDB"); - append_cmd_cnt++; - redisAppendCommand(ctx,"SET MAAT_VERSION %lld",maat_redis_version); - append_cmd_cnt++; - redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version); - append_cmd_cnt++; - redisAppendCommand(ctx,"SET %s 1", mr_region_id_var); - append_cmd_cnt++; - redisAppendCommand(ctx,"SET %s 1", mr_group_id_var); - append_cmd_cnt++; - redisAppendCommand(ctx,"EXEC"); - append_cmd_cnt++; - for(i=0;iqueue); - batch->feather=(struct _Maat_feather_t *)feather; - redisContext* write_ctx=get_redis_ctx_for_write(batch->feather); - if(write_ctx==NULL) - { - free(batch); - return NULL; - } - batch->server_time=redis_server_time(write_ctx); - if(!batch->server_time) - { - free(batch); - return NULL; - } - return batch; -} - -int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id) -{ - struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1); - long long absolute_expire_time=0; - char line[MAX_TABLE_LINE_SIZE]; - - serialize_region(region, group_id, line, sizeof(line)); - - set_serial_rule(s_rule, op, region->region_id, 0, region->table_name, - line, absolute_expire_time); - TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries); - batch->batch_size++; - return 0; - -} -#define TO_GROUP2X_KEY(group_id, parent_id) ((unsigned long)group_id<<32|parent_id) - -int Maat_command_batch_set_group2group(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g) -{ - struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1); - long long absolute_expire_time=0; - char line[MAX_TABLE_LINE_SIZE]; - - serialize_group2group(op, g2g, line, sizeof(line)); - - set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2g->group_id, g2g->superior_group_id), 0, g2g->table_name, - line, absolute_expire_time); - - TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries); - batch->batch_size++; - return 0; -} -int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c) -{ - struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1); - long long absolute_expire_time=0; - char line[MAX_TABLE_LINE_SIZE]; - - serialize_group2compile(op, g2c, line, sizeof(line)); - set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2c->group_id, g2c->compile_id), 0, g2c->table_name, - line, absolute_expire_time); - TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries); - batch->batch_size++; - return 0; -} -int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after) -{ - - struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1); - long long absolute_expire_time=0; - char line[MAX_TABLE_LINE_SIZE]; - serialize_compile(compile, huge_service_defined, clause_num, op, line, sizeof(line)); - - if(expire_after>0) - { - absolute_expire_time=batch->server_time+expire_after; - } - set_serial_rule(s_rule, op, compile->config_id, label_id, table_name, - line, absolute_expire_time); - - TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries); - batch->batch_size++; - return 0; -} -int Maat_command_batch_commit(struct Maat_command_batch* batch) -{ - struct serial_rule_t* s_rule_array=ALLOC(struct serial_rule_t, batch->batch_size); - int i=0; - redisContext* write_ctx=get_redis_ctx_for_write(batch->feather); - struct serial_rule_t * tmp = TAILQ_FIRST(&batch->queue); - - while(tmp != NULL) - { - TAILQ_REMOVE(&batch->queue, tmp, entries); - memcpy(s_rule_array+i, tmp, sizeof(*tmp)); - free(tmp); - tmp = TAILQ_FIRST(&batch->queue); - i++; - } - assert(i==batch->batch_size); - exec_serial_rule(write_ctx, s_rule_array, batch->batch_size, batch->server_time, batch->feather->logger); - for(i=0; ibatch_size; i++) - { - empty_serial_rules(s_rule_array+i); - } - free(s_rule_array); - free(batch); - return i; -} - -int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after) -{ - struct Maat_command_batch* batch=NULL; - batch=Maat_command_batch_new(feather); - Maat_command_batch_set_compile(batch, op, compile, table_name, huge_service_defined, clause_num, label_id, expire_after); - Maat_command_batch_commit(batch); - return 0; -} -int Maat_command_raw_set_region(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id) -{ - struct Maat_command_batch* batch=NULL; - batch=Maat_command_batch_new(feather); - Maat_command_batch_set_region(batch, op, region, group_id); - Maat_command_batch_commit(batch); - return 0; -} -int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c) -{ - struct Maat_command_batch* batch=NULL; - batch=Maat_command_batch_new(feather); - Maat_command_batch_set_group2compile(batch, op, g2c); - Maat_command_batch_commit(batch); - return 0; -} -int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g) -{ - - struct Maat_command_batch* batch=NULL; - batch=Maat_command_batch_new(feather); - Maat_command_batch_set_group2group(batch, op, g2g); - Maat_command_batch_commit(batch); - return 0; -} - -int Maat_cmd_flushDB(Maat_feather_t feather) -{ - _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=0; - redisContext* write_ctx=get_redis_ctx_for_write(_feather); - if(write_ctx==NULL) - { - return -1; - } - do - { - ret=redis_flush_DB(_feather->mr_ctx.write_ctx, _feather->mr_ctx.redis_db, _feather->logger); - }while(ret==0); - return 0; -} - diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp deleted file mode 100644 index ec73c61..0000000 --- a/src/entry/Maat_ex_data.cpp +++ /dev/null @@ -1,397 +0,0 @@ -#include "Maat_ex_data.h" -#include "Maat_table_schema.h" -#include "Maat_utils.h" -#include "Maat_garbage_collection.h" - -#include "uthash/uthash.h" -#include "uthash/utarray.h" - -#include -#include - - -struct EX_data_container -{ - void* ex_data; - char* key; - size_t key_len; - void* user_data; - - UT_hash_handle hh_a, hh_b; - const struct EX_data_rt* rt; - -}; - -struct EX_data_rt -{ - UT_array *cache_rows; - size_t cache_row_num; - size_t cache_size; - - pthread_mutex_t mutex_update_commit; - int is_updating; - char effective_hash;//value 'a' or 'b', indicates which hash is effective - struct EX_data_container* hash_key2ex_a, *hash_key2ex_b; //two hash for read-copy-update (RCU) - - const struct EX_data_schema* ex_schema; - int table_id; - void (* user_data_free)(void *user_data); - struct Maat_garbage_bin* garbage_bin; -}; - - -void EX_data_container_free(struct EX_data_container* ex_container) -{ - const struct EX_data_schema* ex_schema=ex_container->rt->ex_schema; - ex_schema->free_func(ex_container->rt->table_id, &(ex_container->ex_data), ex_schema->argl, ex_schema->argp); - if(ex_container->user_data && ex_container->rt->user_data_free) - { - ex_container->rt->user_data_free(ex_container->user_data); - } - free(ex_container->key); - ex_container->key=NULL; - ex_container->key_len=0; - ex_container->user_data=NULL; - ex_container->rt=NULL; - free(ex_container); - return; -} -void cache_row_free(void*p) -{ - free(*(char**)p); -} -UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; - -struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) -{ - struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1); - p->hash_key2ex_a=NULL; - p->hash_key2ex_b=NULL; - p->effective_hash='a'; - utarray_new(p->cache_rows, &ut_cache_row_icd); - p->table_id=table_id; - p->user_data_free=user_data_free; - p->garbage_bin=Maat_garbage_bin_new(60); - pthread_mutex_init(&p->mutex_update_commit, NULL); - return p; -}; -size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt) -{ - return ex_rt->cache_row_num; -} -void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema) -{ - p->ex_schema=schema; -} -void EX_data_rt_free(struct EX_data_rt* ex_rt) -{ - struct EX_data_container* ex_container=NULL, *tmp=NULL; - - assert(ex_rt->is_updating==0); - if(ex_rt->effective_hash=='a') - { - HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp) - { - HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, ex_container); - EX_data_container_free(ex_container); - } - } - else - { - HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp) - { - HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, ex_container); - EX_data_container_free(ex_container); - } - - } - //ex_rt->hash_key2ex's memory is freed when its last element was deleted. - - if(ex_rt->cache_rows) - { - utarray_free(ex_rt->cache_rows); - ex_rt->cache_rows=NULL; - ex_rt->cache_row_num=0; - } - Maat_garbage_bin_free(ex_rt->garbage_bin); - ex_rt->garbage_bin=NULL; - free(ex_rt); - return; -} -void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row) -{ - size_t len=strlen(row)+1; - char* row_copy=ALLOC(char, len); - memcpy(row_copy, row, len); - p->cache_size+=len; - utarray_push_back(p->cache_rows, &row_copy); - p->cache_row_num++; - return; -} -const char* EX_data_rt_cached_row_get(struct EX_data_rt* p, size_t index) -{ - const char** row=NULL; - row=(const char**)utarray_eltptr(p->cache_rows, index); - return *row; -} -void EX_data_rt_clear_row_cache(struct EX_data_rt* p) -{ - utarray_free(p->cache_rows); - p->cache_rows=NULL; - p->cache_row_num=0; - p->cache_size=0; -} -void EX_data_rt_update_prepare(struct EX_data_rt* ex_rt) -{ - struct EX_data_container* ex_container=NULL, *tmp=NULL; - if(ex_rt->effective_hash=='a') - { - assert(ex_rt->hash_key2ex_b==NULL); - HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp) - { - HASH_ADD_KEYPTR(hh_b, ex_rt->hash_key2ex_b, ex_container->key, ex_container->key_len, ex_container); - } - } - else - { - assert(ex_rt->hash_key2ex_a==NULL); - HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp) - { - HASH_ADD_KEYPTR(hh_a, ex_rt->hash_key2ex_a, ex_container->key, ex_container->key_len, ex_container); - } - } - assert(0==Maat_garbage_bin_get_size(ex_rt->garbage_bin)); - ex_rt->is_updating=1; - return; -} - -void EX_data_rt_update_commit(struct EX_data_rt* ex_rt) -{ - struct EX_data_container* ex_container=NULL, *tmp=NULL; - //lock contention of Maat_plugin_EX_register and rule update thread. - pthread_mutex_lock(&ex_rt->mutex_update_commit); - if(!ex_rt->is_updating) - { - pthread_mutex_unlock(&ex_rt->mutex_update_commit); - return; - } - if(ex_rt->effective_hash=='a') - { - ex_rt->effective_hash='b'; - usleep(100); //urgly sleep, wait for EX_data_rt_get_EX_data_by_key to release the effective hash table. - HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp) - { - HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, ex_container); - //ex_container is no need to free, it's either in new hash table or garbage collection queue. - } - } - else - { - ex_rt->effective_hash='a'; - usleep(100); - HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp) - { - HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, ex_container); - } - } - ex_rt->is_updating=0; - Maat_garbage_collect_by_force(ex_rt->garbage_bin); - pthread_mutex_unlock(&ex_rt->mutex_update_commit); - return; -} - -struct EX_data_container* EX_data_rt_effective_hash_find(struct EX_data_rt* ex_rt, const char* key, size_t key_len) -{ - struct EX_data_container* exc=NULL; - - if(ex_rt->effective_hash=='a') - { - HASH_FIND(hh_a, ex_rt->hash_key2ex_a, key, key_len, exc); - } - else - { - assert(ex_rt->effective_hash=='b'); - HASH_FIND(hh_b, ex_rt->hash_key2ex_b, key, key_len, exc); - } - return exc; -} - -struct EX_data_container* EX_data_rt_updating_hash_find(struct EX_data_rt* ex_rt, const char* key, size_t key_len) -{ - struct EX_data_container* exc=NULL; - - if(ex_rt->effective_hash=='a') - { - HASH_FIND(hh_b, ex_rt->hash_key2ex_b, key, key_len, exc); - } - else - { - assert(ex_rt->effective_hash=='b'); - HASH_FIND(hh_a, ex_rt->hash_key2ex_a, key, key_len, exc); - } - return exc; -} -void EX_data_rt_updating_hash_add(struct EX_data_rt* ex_rt, const char* key, size_t key_len, struct EX_data_container* exc) -{ - if(ex_rt->effective_hash=='a') - { - HASH_ADD_KEYPTR(hh_b, ex_rt->hash_key2ex_b, key,key_len, exc); - } - else - { - HASH_ADD_KEYPTR(hh_a, ex_rt->hash_key2ex_a, key,key_len, exc); - } - return; -} -void EX_data_rt_updating_hash_del(struct EX_data_rt* ex_rt, struct EX_data_container* exc) -{ - if(ex_rt->effective_hash=='a') - { - HASH_DELETE(hh_b, ex_rt->hash_key2ex_b, exc); - } - else - { - HASH_DELETE(hh_a, ex_rt->hash_key2ex_a, exc); - } - return; -} -int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, - const char* row, const char* key, size_t key_len, - void* user_data, void* logger) -{ - - MAAT_RULE_EX_DATA ex_data=NULL; - const struct EX_data_schema* ex_schema=ex_rt->ex_schema; - struct EX_data_container* ex_container=NULL, *tmp=NULL; - int ret=0; - if(!ex_rt->is_updating) - { - EX_data_rt_update_prepare(ex_rt); - } - tmp=EX_data_rt_updating_hash_find(ex_rt, key, key_len); - if(tmp==NULL) - { - ex_container=ALLOC(struct EX_data_container, 1); - ex_container->key=ALLOC(char, key_len+1); - memcpy(ex_container->key, key, key_len); - ex_schema->new_func(ex_rt->table_id, ex_container->key, row, &ex_data, - ex_schema->argl, ex_schema->argp); - ex_container->ex_data=ex_data; - ex_container->rt=ex_rt; - ex_container->user_data=user_data; - - ex_container->key_len=key_len; - EX_data_rt_updating_hash_add(ex_rt, ex_container->key, ex_container->key_len, ex_container); - ret=0; - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "EX data add error: duplicated key %.*s of %s", - key_len, key, row); - - ret=-1; - } - return ret; -} -int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, - void *logger) -{ - struct EX_data_container* exc=NULL; - int ret=0; - if(!ex_rt->is_updating) - { - EX_data_rt_update_prepare(ex_rt); - } - exc=EX_data_rt_updating_hash_find(ex_rt, key, key_len); - if(exc) - { - EX_data_rt_updating_hash_del(ex_rt, exc); - Maat_garbage_bagging(ex_rt->garbage_bin, exc, (void (*)(void*))EX_data_container_free); - ret=0; - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "EX data del error: no such key %.*s of %s", - key_len, key, row); - ret=-1; - } - return ret; -} - -MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len) -{ - struct EX_data_container* exc=NULL; - MAAT_RULE_EX_DATA ex_data=NULL; - - if(!ex_rt->ex_schema) - { - assert(0); - return NULL; - } - exc=EX_data_rt_effective_hash_find(ex_rt, key, key_len); - if(exc!=NULL) - { - ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(exc->ex_data), - ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); - } - return ex_data; -} -MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container) -{ - MAAT_RULE_EX_DATA dupped_ex_data=NULL; - ex_rt->ex_schema->dup_func(ex_rt->table_id, &(dupped_ex_data), &(container->ex_data), - ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); - return dupped_ex_data; -} - -size_t EX_data_rt_list_updating_ex_containers(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array) -{ - size_t ex_data_cnt=0, i=0; - struct EX_data_container* ex_container=NULL, *tmp=NULL; - if(!ex_rt->ex_schema) - { - return 0; - } - assert(ex_rt->is_updating==1); - if(ex_rt->effective_hash=='a') - { - ex_data_cnt=HASH_CNT(hh_b, ex_rt->hash_key2ex_b); - *ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt); - HASH_ITER(hh_b, ex_rt->hash_key2ex_b, ex_container, tmp) - { - (*ex_container_array)[i]=ex_container; - i++; - } - } - else - { - ex_data_cnt=HASH_CNT(hh_a, ex_rt->hash_key2ex_a); - *ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt); - HASH_ITER(hh_a, ex_rt->hash_key2ex_a, ex_container, tmp) - { - (*ex_container_array)[i]=ex_container; - i++; - } - } - return ex_data_cnt; -} -void* EX_data_container_get_user_data(struct EX_data_container* ex_container) -{ - return ex_container->user_data; -} -size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt) -{ - size_t count=0; - if(ex_rt->effective_hash=='a') - { - count=HASH_CNT(hh_a, ex_rt->hash_key2ex_a); - } - else - { - count=HASH_CNT(hh_b, ex_rt->hash_key2ex_b); - } - return count; -} - diff --git a/src/entry/Maat_garbage_collection.cpp b/src/entry/Maat_garbage_collection.cpp deleted file mode 100644 index ff981bf..0000000 --- a/src/entry/Maat_garbage_collection.cpp +++ /dev/null @@ -1,96 +0,0 @@ - -#include "Maat_utils.h" - -#include -#include -#include -#include - -struct Maat_garbage_bag -{ - time_t create_time; - int timeout; - int ok_times; - void *garbage; - void (* garbage_free)(void *garbage); - TAILQ_ENTRY(Maat_garbage_bag) entries; -}; -TAILQ_HEAD(Maat_garbage_q, Maat_garbage_bag); - -struct Maat_garbage_bin -{ - Maat_garbage_q garbage_q; - size_t bag_cnt; - int timeout_seconds; -}; -struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout) -{ - struct Maat_garbage_bin* bin=ALLOC(struct Maat_garbage_bin, 1); - TAILQ_INIT(&bin->garbage_q); - bin->timeout_seconds=default_timeout; - return bin; -} -void Maat_garbage_bin_free(struct Maat_garbage_bin* bin) -{ - struct Maat_garbage_bag* p=NULL; - while (p=TAILQ_FIRST(&bin->garbage_q)) - { - p->garbage_free(p->garbage); - TAILQ_REMOVE(&bin->garbage_q, p, entries); - free(p); - bin->bag_cnt--; - } - - free(bin); - return; -} -size_t Maat_garbage_bin_get_size(struct Maat_garbage_bin* bin) -{ - return bin->bag_cnt; -} -void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)) -{ - struct Maat_garbage_bag* bag=ALLOC( struct Maat_garbage_bag, 1); - bag->create_time=time(NULL); - bag->timeout=bin->timeout_seconds; - bag->garbage=garbage; - bag->garbage_free=func; - TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries); - bin->bag_cnt++; -} -void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin) -{ - struct Maat_garbage_bag* p=NULL, *tmp=NULL; - size_t n_clollected=0, n_bag=0; - time_t now=time(NULL); - - for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp) - { - tmp=TAILQ_NEXT(p, entries); - if(now-p->create_time>p->timeout || p->timeout==0) - { - p->garbage_free(p->garbage); - TAILQ_REMOVE(&bin->garbage_q, p, entries); - free(p); - n_clollected++; - } - n_bag++; - } - - assert(bin->bag_cnt==n_bag); - bin->bag_cnt-=n_clollected; - return; -} -void Maat_garbage_collect_by_force(struct Maat_garbage_bin* bin) -{ - struct Maat_garbage_bag* p=NULL; - while (p=TAILQ_FIRST(&bin->garbage_q)) - { - p->garbage_free(p->garbage); - TAILQ_REMOVE(&bin->garbage_q, p, entries); - free(p); - bin->bag_cnt--; - } - return; -} - diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp deleted file mode 100644 index 2188c6e..0000000 --- a/src/entry/Maat_hierarchy.cpp +++ /dev/null @@ -1,1550 +0,0 @@ - -#include "Maat_hierarchy.h" -#include "Maat_utils.h" -#include "Maat_limits.h" -#include "uthash/uthash.h" -#include "uthash/utarray.h" -#include "igraph/igraph.h" -#include "bool_matcher.h" - -#include - -#include -#include - -#define module_maat_hierarchy "MAAT_HIERARCHY" - - - -struct Maat_hierarchy_group -{ - igraph_integer_t vertex_id; - int group_id; - int ref_by_compile_cnt; - int ref_by_superior_group_cnt; - int ref_by_subordinate_group_cnt; - int ref_by_region_cnt; - - size_t top_group_cnt; - int* top_group_ids; - UT_hash_handle hh_group_id; - UT_hash_handle hh_vertex_id; -}; -struct Maat_hierarchy_region -{ - int region_id; - int group_id; - int table_id; - - struct Maat_hierarchy_group* ref_parent_group; - UT_hash_handle hh; - - void* user_data; -}; - -struct Maat_hierarchy_literal_id -{ - int group_id; - int vt_id; -}; - -struct Maat_hierarchy_literal -{ - struct Maat_hierarchy_literal_id literal_id; - UT_array *clause_ids; - UT_hash_handle hh; //index to -}; -struct Maat_hierarchy_clause_state -{ - - unsigned long long clause_id; - char not_flag; - char in_use; - UT_array *literal_ids; -}; - -struct Maat_hierarchy_internal_hit_path -{ - int Nth_scan; - int Nth_hit_region; - int region_id; - int virtual_table_id; -}; -static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result) -{ - struct Maat_hierarchy_internal_hit_path new_path; - new_path.region_id=region_id; - new_path.Nth_hit_region=Nth_region_result; - new_path.Nth_scan=Nth_scan; - new_path.virtual_table_id=virtual_table_id; - /* - struct Maat_hierarchy_internal_hit_path *tmp_path=NULL; - size_t i=0, num=utarray_len(hit_paths); - for(i=0; iNth_scan!=new_path.Nth_scan) - { - break; - } - else - { - if(tmp_path->region_id==new_path.region_id && - tmp_path->virtual_table_id==new_path.virtual_table_id && - tmp_path->Nth_hit_region==new_path.Nth_hit_region) - { - return 0; - } - } - } - */ - utarray_push_back(hit_paths, &new_path); - return 1; -} - -UT_icd ut_literal_id_icd = {sizeof(struct Maat_hierarchy_literal_id), NULL, NULL, NULL}; -UT_icd ut_clause_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; -UT_icd ut_region_id_icd = {sizeof(int), NULL, NULL, NULL}; -UT_icd ut_hit_path_icd = {sizeof(struct Maat_hierarchy_internal_hit_path), NULL, NULL, NULL}; - -#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d -struct Maat_hierarchy_compile -{ - unsigned int magic; - int compile_id; - int actual_clause_num; - int declared_clause_num; - int not_clause_cnt; - void* user_data; - UT_hash_handle hh; - struct Maat_hierarchy_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; -}; - -static void _group_vertex_free(struct Maat_hierarchy_group* group) -{ - free(group->top_group_ids); - free(group); -} - -struct Maat_hierarchy_clause -{ - long long clause_id; - size_t n_literal_id; - struct Maat_hierarchy_literal_id* literal_ids; - UT_hash_handle hh; -}; - -struct group2region -{ - int group_id; - UT_array* region_ids; - UT_hash_handle hh; //index to -}; -struct region2clause_key -{ - int region_id; - int vt_id; -}; -struct region2clause_value -{ - struct region2clause_key key; - UT_array* clause_ids; - int group_id; - UT_hash_handle hh; //index to -}; -void Maat_hierarchy_free_region2clause_hash(struct region2clause_value* hash) -{ - struct region2clause_value* r2c_val=NULL, *tmp_r2c_val=NULL; - HASH_ITER(hh, hash, r2c_val, tmp_r2c_val) - { - HASH_DEL(hash, r2c_val); - utarray_free(r2c_val->clause_ids); - free(r2c_val); - } - assert(hash==NULL); - return; -} - -struct Maat_hierarchy -{ - pthread_rwlock_t rwlock; - pthread_mutex_t mutex; - time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. - int changed_flag; - - struct Maat_hierarchy_compile* hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*. - void (* compile_user_data_free)(void *compile_ud); - - struct Maat_hierarchy_group* hash_group_by_id; //key: group_id, value: struct Maat_hierarchy_group*. - struct Maat_hierarchy_group* hash_group_by_vertex; //key:vetex_id, value: struct Maat_hierarchy_group*. Multimap (Items with multiple keys). - - - struct Maat_hierarchy_region* hash_region_by_id; //key: region_id, value: struct Maat_hierarchy_region*. - - struct Maat_hierarchy_clause* hash_dedup_clause_by_literals; //key: literal combination, value: struct Maat_hierarchy_clause*. For generating unique clause_id. - unsigned long long clause_id_generator; //Increasing number. - - void (* region_user_data_free)(void *region_ud); - - - igraph_t group_graph; - igraph_integer_t group_graph_vcount; - igraph_vector_t dfs_vids; - - igraph_integer_t grp_vertex_id_generator; - - /*Following members are accessed from scan threads.*/ - struct region2clause_value* hash_region2clause; //key: region_id+virtual_table_id, value: struct region2clause_value. - struct bool_matcher* bm; - - - int thread_num; - struct Maat_garbage_bin* ref_garbage_bin; - void* logger; - struct bool_expr_match *expr_match_buff; -}; - -int compare_literal_id(const void *pa, const void *pb) -{ - struct Maat_hierarchy_literal_id *la=(struct Maat_hierarchy_literal_id *)pa; - struct Maat_hierarchy_literal_id *lb=(struct Maat_hierarchy_literal_id *)pb; - int ret=la->vt_id-lb->vt_id; - if(ret==0) - { - ret=la->group_id-lb->group_id; - } - return ret; - -} -static inline int compare_clause_id(const void* a, const void* b) -{ - long long ret=*(const unsigned long long *)a - *(const unsigned long long *)b; - if(ret==0) - { - return 0; - } - else if(ret<0) - { - return -1; - } - else - { - return 1; - } -} -static inline int compare_region_id(const void* a, const void* b) -{ - int ret= *(int*)a - *(int*)b; - return ret; -} - -static const struct Maat_hierarchy_clause* Maat_hierarchy_clause_fetch(struct Maat_hierarchy* hier, struct Maat_hierarchy_literal_id* literal_ids, size_t n_literal_id) -{ - struct Maat_hierarchy_clause* clause=NULL; - - HASH_FIND(hh, hier->hash_dedup_clause_by_literals, literal_ids, - n_literal_id*sizeof(struct Maat_hierarchy_literal_id), clause); - - if(!clause) - { - clause=ALLOC(struct Maat_hierarchy_clause, 1); - clause->clause_id=hier->clause_id_generator; - clause->n_literal_id=n_literal_id; - clause->literal_ids=ALLOC(struct Maat_hierarchy_literal_id, n_literal_id); - memcpy(clause->literal_ids, literal_ids, n_literal_id*sizeof(struct Maat_hierarchy_literal_id)); - - hier->clause_id_generator++; - HASH_ADD_KEYPTR(hh, hier->hash_dedup_clause_by_literals, clause->literal_ids, - n_literal_id*sizeof(struct Maat_hierarchy_literal_id), - clause); - } - return clause; -} -static void Maat_hierarchy_clause_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_clause* clause) -{ - HASH_DELETE(hh, hier->hash_dedup_clause_by_literals, clause); - free(clause->literal_ids); - clause->n_literal_id=0; - free(clause); - return; -} - -static int Maat_hierarchy_compile_add_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id, int not_flag, int clause_index) -{ - struct Maat_hierarchy_clause_state* clause_state=compile->clause_states+clause_index; - struct Maat_hierarchy_literal_id* tmp=NULL; - clause_state->not_flag=not_flag; - if(!clause_state->in_use) - { - clause_state->in_use=1; - compile->actual_clause_num++; - } - tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); - if(tmp) - { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); - return -1; - } - else - { - utarray_push_back(clause_state->literal_ids, literal_id); - utarray_sort(clause_state->literal_ids, compare_literal_id); - } - return 0; -} -static int Maat_hierarchy_compile_remove_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id, int clause_index) -{ - struct Maat_hierarchy_clause_state* clause_state=compile->clause_states+clause_index; - struct Maat_hierarchy_literal_id* tmp=NULL; - size_t remove_idx=0; - tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id , compare_literal_id); - if(tmp) - { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); - } - else - { - return -1; - } - remove_idx=utarray_eltidx(clause_state->literal_ids, tmp); - utarray_erase(clause_state->literal_ids, remove_idx, 1); - if(0==utarray_len(clause_state->literal_ids)) - { - clause_state->in_use=0; - compile->actual_clause_num--; - } - return 0; -} -static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hierarchy* hier, int compile_id) -{ - int i=0; - struct Maat_hierarchy_compile* compile=NULL; - compile=ALLOC(struct Maat_hierarchy_compile, 1); - compile->magic=MAAT_HIER_COMPILE_MAGIC; - compile->compile_id=compile_id; - HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile); - for(i=0; iclause_states[i].literal_ids, &ut_literal_id_icd); - compile->clause_states[i].in_use=0; - } - return compile; -} -static void Maat_hierarchy_compile_free(struct Maat_hierarchy_compile* compile) -{ - int i=0; - struct Maat_hierarchy_clause_state* clause_state=NULL; - //user_data must be freed before calling this function. - assert(compile->user_data==NULL); - for(i=0; iclause_states+i; - utarray_free(clause_state->literal_ids); - clause_state->literal_ids=NULL; - clause_state->in_use=0; - } - compile->magic=0; - free(compile); -} - -static struct Maat_hierarchy_region* Maat_hierarchy_region_new(struct Maat_hierarchy* hier, int region_id, int group_id, int table_id, struct Maat_hierarchy_group* parent_group, void* user_data) -{ - struct Maat_hierarchy_region* region=NULL; - region=ALLOC(struct Maat_hierarchy_region, 1); - region->group_id=group_id; - region->region_id=region_id; - region->table_id=table_id; - region->ref_parent_group=parent_group; - region->user_data=user_data; - HASH_ADD_INT(hier->hash_region_by_id, region_id, region); - parent_group->ref_by_region_cnt++; - return region; -} -static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_region* region) -{ - HASH_DELETE(hh, hier->hash_region_by_id, region); - region->ref_parent_group->ref_by_region_cnt--; - if(hier->region_user_data_free && region->user_data) - { - hier->region_user_data_free(region->user_data); - region->user_data=NULL; - } - free(region); - return; -} - - -struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger, struct Maat_garbage_bin* bin) -{ - struct Maat_hierarchy* hier=ALLOC(struct Maat_hierarchy, 1); - UNUSED int ret=0; - hier->logger=mesa_handle_logger; - hier->thread_num=thread_num; - hier->version=time(NULL); - - hier->hash_group_by_id=NULL; - hier->hash_group_by_vertex=NULL; - hier->hash_compile_by_id=NULL; - hier->hash_region2clause=NULL; - hier->hash_region_by_id=NULL; - hier->hash_dedup_clause_by_literals=NULL; - hier->clause_id_generator=0; - hier->ref_garbage_bin=bin; - hier->expr_match_buff=ALLOC(struct bool_expr_match, thread_num*MAX_SCANNER_HIT_NUM); - - pthread_mutex_init(&hier->mutex, NULL); - ret=pthread_rwlock_init(&hier->rwlock, NULL); - assert(ret==0); - ret=igraph_empty(&hier->group_graph, 0, IGRAPH_DIRECTED); - assert(ret==IGRAPH_SUCCESS); - return hier; -} -void Maat_hierarchy_free(struct Maat_hierarchy* hier) -{ - struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; - struct Maat_hierarchy_group* group=NULL, *tmp_group=NULL; - struct region2clause_value* r2c_val=NULL, *tmp_r2c_val=NULL; - struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL; - struct Maat_hierarchy_clause* clause=NULL, *tmp_clause=NULL; - pthread_rwlock_wrlock(&hier->rwlock); - - //Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do - //Some have asked how uthash cleans up its internal memory. - //The answer is simple: when you delete the final item from a hash table, - //uthash releases all the internal memory associated with that hash table, - //and sets its pointer to NULL. - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) - { - if(hier->compile_user_data_free && compile->user_data) - { - hier->compile_user_data_free(compile->user_data); - compile->user_data=NULL; - } - HASH_DEL(hier->hash_compile_by_id, compile); - Maat_hierarchy_compile_free(compile); - } - assert(hier->hash_compile_by_id==NULL); - - HASH_ITER(hh, hier->hash_region2clause, r2c_val, tmp_r2c_val) - { - HASH_DEL(hier->hash_region2clause, r2c_val); - utarray_free(r2c_val->clause_ids); - free(r2c_val); - } - Maat_hierarchy_free_region2clause_hash(hier->hash_region2clause); - - HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) - { - Maat_hierarchy_region_free(hier, region); - } - - HASH_ITER(hh, hier->hash_dedup_clause_by_literals, clause, tmp_clause) - { - Maat_hierarchy_clause_free(hier, clause); - } - - //Free group as the last. - HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up. - HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group) - { - HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); - _group_vertex_free(group); - } - assert(hier->hash_group_by_id==NULL); - - igraph_destroy(&hier->group_graph); - bool_matcher_free(hier->bm); - hier->bm=NULL; - pthread_rwlock_unlock(&hier->rwlock); - pthread_rwlock_destroy(&hier->rwlock); - free(hier->expr_match_buff); - hier->expr_match_buff=NULL; - free(hier); -} - -void Maat_hierarchy_set_compile_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)) -{ - hier->compile_user_data_free=func; - return; -} -void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)) -{ - hier->region_user_data_free=func; - return; -} - - -int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data) -{ - int ret=0; - struct Maat_hierarchy_compile* compile=NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if(!compile) - { - assert(declared_clause_num>=0); - compile=Maat_hierarchy_compile_new(hier, compile_id); - compile->declared_clause_num=declared_clause_num; - compile->user_data=user_data; - } - else - { - if(compile->user_data!=NULL) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Add compile %d failed, compile is already exisited.", - compile_id); - ret=-1; - } - else - { - compile->declared_clause_num=declared_clause_num; - compile->user_data=user_data; - } - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; -} -int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id) -{ - struct Maat_hierarchy_compile* compile=NULL; - int ret=0; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if(compile) - { - if(hier->compile_user_data_free && compile->user_data) - { - hier->compile_user_data_free(compile->user_data); - compile->user_data=NULL; - } - if(compile->actual_clause_num==0) - { - HASH_DEL(hier->hash_compile_by_id, compile); - Maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))Maat_hierarchy_compile_free); - } - ret=0; - } - else - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove compile %d failed, compile is not exisited.", - compile_id); - ret=-1; - - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; -} -static void* Maat_hier_compile_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach) -{ - struct Maat_hierarchy_compile* compile=NULL; - void* ret=NULL; - - pthread_rwlock_rdlock(&hier->rwlock); - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if(compile) - { - ret=compile->user_data; - if(is_dettach) - { - compile->user_data=NULL; - } - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; - -} -void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id) -{ - void* user_data=NULL; - user_data=Maat_hier_compile_get_user_data(hier, compile_id, 1); - return user_data; -} -void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id) -{ - void* user_data=NULL; - user_data=Maat_hier_compile_get_user_data(hier, compile_id, 0); - return user_data; - -} -void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param) -{ - struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; - pthread_rwlock_rdlock(&hier->rwlock); - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) - { - if(compile->user_data) - { - callback(compile->user_data, param); - } - } - pthread_rwlock_unlock(&hier->rwlock); - return; -} -struct Maat_hierarchy_group* Maat_hierarchy_group_new(struct Maat_hierarchy* hier, int group_id) -{ - struct Maat_hierarchy_group* group=NULL; - group=ALLOC(struct Maat_hierarchy_group, 1); - group->group_id=group_id; - group->vertex_id=hier->grp_vertex_id_generator++; - assert(igraph_vcount(&hier->group_graph)==group->vertex_id); - igraph_add_vertices(&hier->group_graph, 1, NULL); //Add 1 vertice. - - HASH_ADD(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group->group_id), group); - HASH_ADD(hh_vertex_id, hier->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); - - return group; -} -void vector_print(igraph_vector_t *v) { - long int i; - for (i=0; iref_by_compile_cnt==0&&group->ref_by_superior_group_cnt==0); - igraph_vector_init(&v, 8); - igraph_neighbors(&hier->group_graph, &v, group->vertex_id, IGRAPH_ALL); - if(igraph_vector_size(&v)>0) - { - print_igraph_vector(&v, buff, sizeof(buff)); - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Del group %d exception, still reached by %s.", - group->vertex_id, buff); - assert(0); - } - igraph_vector_destroy(&v); - assert(group->top_group_ids==NULL); - //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices. - //igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id)); - - - HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); - HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group); - - _group_vertex_free(group); - - return; -} - -int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id) -{ - int ret=0; - struct Maat_hierarchy_group* group=NULL; - struct Maat_hierarchy_literal_id literal_id={group_id, vt_id}; - struct Maat_hierarchy_compile* compile=NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(!group) - { - group=Maat_hierarchy_group_new(hier, group_id); - } - - HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); - if(!compile) - { - compile=Maat_hierarchy_compile_new(hier, compile_id); - } - - ret=Maat_hierarchy_compile_add_literal(compile, &literal_id, not_flag, clause_index); - if(ret<0) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.", - group_id, vt_id, clause_index, compile_id); - ret=-1; - } - else - { - ret=0; - group->ref_by_compile_cnt++; - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; -} -int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id) -{ - struct Maat_hierarchy_group* group=NULL; - struct Maat_hierarchy_literal_id literal_id={group_id, vt_id}; - struct Maat_hierarchy_compile* compile=NULL; - int ret=0; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(!group) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove group %d from compile %d failed, group is not exisited.", - group_id, compile_id); - goto error_out; - } - HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); - if(!compile) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove group %d from compile %d failed, compile is not exisited.", - group_id, compile_id); - goto error_out; - } - - ret=Maat_hierarchy_compile_remove_literal(compile, &literal_id, clause_index); - if(ret<0) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", - group_id, vt_id, clause_index, compile_id); - goto error_out; - } - if(compile->actual_clause_num==0 && !compile->user_data) - { - HASH_DEL(hier->hash_compile_by_id, compile); - Maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))Maat_hierarchy_compile_free); - } - pthread_rwlock_unlock(&hier->rwlock); - return 0; - -error_out: - pthread_rwlock_unlock(&hier->rwlock); - return -1; -} - - -int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) -{ - int ret=0; - igraph_integer_t edge_id; - struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(!group) - { - group=Maat_hierarchy_group_new(hier, group_id); - } - - HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); - if(!superior_group) - { - superior_group=Maat_hierarchy_group_new(hier, superior_group_id); - } - ret=igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); - if(edge_id>0)//No duplicated edges between two groups. - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Add group %d to group %d failed, relation already exisited.", - group->group_id, superior_group->group_id); - ret=-1; - } - else - { - igraph_add_edge(&hier->group_graph, group->vertex_id, superior_group->vertex_id); - group->ref_by_superior_group_cnt++; - superior_group->ref_by_subordinate_group_cnt++; - ret=0; - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; -} - -int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id) -{ - int ret=0; - struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; - - //No hash write operation, LOCK protection is unnecessary. - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(group==NULL) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Del group %d from group %d failed, group %d not exisited.", - group_id, superior_group_id, group_id); - return -1; - } - HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); - if(superior_group==NULL) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Del group %d from group %d failed, superior group %d not exisited.", - group_id, superior_group_id, superior_group_id); - return -1; - } - igraph_es_t es; - igraph_integer_t edge_num_before=0, edge_num_after=0; - - edge_num_before=igraph_ecount(&hier->group_graph); - // The edges between the given pairs of vertices will be included in the edge selection. - //The vertex pairs must be given as the arguments of the function call, the third argument - //is the first vertex of the first edge, the fourth argument is the second vertex of the - //first edge, the fifth is the first vertex of the second edge and so on. The last element - //of the argument list must be -1 to denote the end of the argument list. - //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); - assert(ret==IGRAPH_SUCCESS); - // ignore no such edge to abort(). - igraph_set_error_handler(igraph_error_handler_ignore); - ret=igraph_delete_edges(&hier->group_graph, es); - edge_num_after=igraph_ecount(&hier->group_graph); - igraph_es_destroy(&es); - - if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1) - { - assert(0); - return -1; - } - - group->ref_by_superior_group_cnt--; - superior_group->ref_by_subordinate_group_cnt--; - return 0; - -} - - -int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data) -{ - //A region rule belongs to ONE group only. - struct Maat_hierarchy_group* group=NULL; - struct Maat_hierarchy_region* region=NULL; - int ret=0; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(!group) - { - group=Maat_hierarchy_group_new(hier, group_id); - } - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if(region) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Add region %d to group %d failed, region already in group %d.", - region_id, - group_id, - region->ref_parent_group->group_id); - ret=-1; - } - else - { - region=Maat_hierarchy_region_new(hier, region_id, group_id, table_id, group, user_data); - ret=0; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} -void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int region_id) -{ - struct Maat_hierarchy_region* region=NULL; - void* ret=NULL; - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if(region) - { - ret=region->user_data; - region->user_data=NULL; - } - pthread_rwlock_unlock(&hier->rwlock); - return ret; -} -int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int group_id, int region_id) -{ - struct Maat_hierarchy_group* group=NULL; - struct Maat_hierarchy_region* region=NULL; - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag=1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if(!group) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove region %d from group %d failed, group is not existed.", - region_id, - group_id); - goto error_out; - } - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if(!region) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Remove region %d from group %d failed, region is not exisited.", - region_id, - group_id); - goto error_out; - } - - assert(region->group_id==group->group_id); - Maat_hierarchy_region_free(hier, region); - pthread_rwlock_unlock(&hier->rwlock); - return 0; - -error_out: - pthread_rwlock_unlock(&hier->rwlock); - return -1; -} - -static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierarchy* hier) -{ - struct bool_matcher* bm=NULL; - size_t compile_num=0, expr_cnt=0; - struct bool_expr* bool_expr_array=NULL; - struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; - - struct Maat_hierarchy_clause_state* clause_state=NULL; - const struct Maat_hierarchy_clause* clause=NULL; - size_t i=0, j=0; - int has_clause_num=0; - compile_num=HASH_COUNT(hier->hash_compile_by_id); - - if(compile_num==0) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "No compile to build."); - return NULL; - } - - //STEP 1, update clause_id of each compile and literal - struct Maat_hierarchy_literal_id* literal_ids=NULL; - size_t n_literal_id=0; - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) - { - has_clause_num=0; - for(i=0; iclause_states+i; - clause_state->clause_id=0; - if(!clause_state->in_use) - { - continue; - } - has_clause_num++; - literal_ids=(struct Maat_hierarchy_literal_id*)utarray_eltptr(clause_state->literal_ids, 0); - n_literal_id=utarray_len(clause_state->literal_ids); - clause=Maat_hierarchy_clause_fetch(hier, literal_ids, n_literal_id); - clause_state->clause_id=clause->clause_id; - } - assert(has_clause_num==compile->actual_clause_num); - } - - //STEP 2, serial compile clause states to a bool expression array. - compile_num=HASH_COUNT(hier->hash_compile_by_id); - bool_expr_array=ALLOC(struct bool_expr, compile_num); - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) - { - for(i=0, j=0; iclause_states[i].in_use) - { - if(compile->clause_states[i].not_flag) - { - compile->not_clause_cnt++; - } - bool_expr_array[expr_cnt].items[j].item_id=compile->clause_states[i].clause_id; - bool_expr_array[expr_cnt].items[j].not_flag=compile->clause_states[i].not_flag; - j++; - } - } - //some compile may have zero groups, e.g. default policy. - if(j==(size_t)compile->declared_clause_num&&j>0) - { - bool_expr_array[expr_cnt].expr_id=compile->compile_id; - bool_expr_array[expr_cnt].user_tag=compile; - bool_expr_array[expr_cnt].item_num=j; - expr_cnt++; - } - } - - //STEP 3, build the bool matcher. - size_t mem_size=0; - if(expr_cnt==0) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "No bool expression to build."); - goto error_out; - } - bm=bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); - if(bm!=NULL) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, - "Build bool matcher of %zu expressions with %zu bytes memory.", - expr_cnt, - mem_size); - } - else - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, module_maat_hierarchy, - "Build bool matcher failed!"); - } - -error_out: - - - free(bool_expr_array); - bool_expr_array=NULL; - - return bm; -} -static int Maat_hierarchy_build_top_groups(struct Maat_hierarchy* hier) -{ - struct Maat_hierarchy_group* group=NULL, *tmp=NULL; - struct Maat_hierarchy_group* superior_group=NULL; - int tmp_vid=0; - size_t i=0, top_group_cnt=0; - int* temp_group_ids=NULL; - - igraph_bool_t is_dag; - igraph_is_dag(&(hier->group_graph), &is_dag); - if(!is_dag) - { - MESA_handle_runtime_log(hier->logger, RLOG_LV_FATAL, maat_module, - "Sub group cycle detected!"); - return -1; - } - hier->group_graph_vcount=igraph_vcount(&hier->group_graph); - igraph_vector_init(&(hier->dfs_vids), hier->group_graph_vcount); - - HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp) - { - top_group_cnt=0; - temp_group_ids=NULL; - //Orphan, Not reference by any one, free it. - if(group->ref_by_compile_cnt==0 - && group->ref_by_superior_group_cnt==0 - && group->ref_by_subordinate_group_cnt==0 - && group->ref_by_region_cnt==0) - { - - free(group->top_group_ids); - group->top_group_ids=NULL; - Maat_hierarchy_group_free(hier, group); - continue; - } - - //A group is need to build top groups when it has regions and referenced by superior groups or compiles. - if(group->ref_by_region_cnt>0 && (group->ref_by_compile_cnt>0 || group->ref_by_superior_group_cnt>0)) - { - if(group->ref_by_superior_group_cnt==0) - { - //fast path, group is only referenced by compile rules. - top_group_cnt=1; - temp_group_ids=ALLOC(int, top_group_cnt); - temp_group_ids[0]=group->group_id; - } - else - { - igraph_vector_t *vids=&(hier->dfs_vids); - igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT, - 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); - - temp_group_ids=ALLOC(int, effective_vertices_count(vids)); - - for(i=0; i<(size_t)igraph_vector_size(vids); i++) - { - tmp_vid=(int) VECTOR(*vids)[i]; - if(tmp_vid<0) - { - break; - } - HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); - if(superior_group->ref_by_compile_cnt>0)//including itself - { - temp_group_ids[top_group_cnt]=superior_group->group_id; - top_group_cnt++; - } - } - } - - } - - free(group->top_group_ids); - group->top_group_cnt=top_group_cnt; - group->top_group_ids=ALLOC(int, group->top_group_cnt); - memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); - - free(temp_group_ids); - temp_group_ids=NULL; - } - igraph_vector_destroy(&hier->dfs_vids); - return 0; -} - -struct region2clause_value* Maat_hierarchy_build_region2clause_hash(struct Maat_hierarchy* hier) -{ - size_t i=0, j=0, k=0; - struct Maat_hierarchy_compile* compile=NULL, *tmp_compile=NULL; - struct Maat_hierarchy_literal_id* literal_id=NULL; - struct Maat_hierarchy_clause_state* clause_state=NULL; - - struct Maat_hierarchy_region* region=NULL, *tmp_region=NULL; - struct Maat_hierarchy_group* group=NULL; - struct group2region* g2r_hash=NULL, *g2r=NULL, *g2r_tmp=NULL; - struct region2clause_value* region2clause_hash=NULL, *r2c_val=NULL; - struct region2clause_key r2c_key; - - //Build a temporary hash that maps group to its regions. - HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) - { - group=region->ref_parent_group; - for(i=0; itop_group_cnt; i++) - { - HASH_FIND_INT(g2r_hash, group->top_group_ids+i, g2r); - if(!g2r) - { - g2r=ALLOC(struct group2region, 1); - utarray_new(g2r->region_ids, &ut_region_id_icd); - utarray_reserve(g2r->region_ids, group->ref_by_region_cnt); - g2r->group_id=group->top_group_ids[i]; - HASH_ADD_INT(g2r_hash, group_id, g2r); - } - - //One region belongs to one group, one group may have many regions. So duplicate region check is unnecessary. - //if(utarray_find(g2r->region_ids, &(region->region_id), compare_region_id)) assert(0); - - utarray_push_back(g2r->region_ids, &(region->region_id)); - //utarray_sort(g2r->region_ids, compare_region_id); - } - } - - //Build short cut hash that maps region_id+vt_id to clause_ids. - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) - { - for(i=0; iclause_states+i; - if(!clause_state->in_use) - { - continue; - } - for(j=0; jliteral_ids); j++) - { - literal_id=(struct Maat_hierarchy_literal_id*)utarray_eltptr(clause_state->literal_ids, j); - HASH_FIND(hh_group_id, hier->hash_group_by_id, &(literal_id->group_id), sizeof(literal_id->group_id), group); - if(!group) - { - continue; - } - HASH_FIND_INT(g2r_hash, &(group->group_id), g2r); - if(!g2r)//group declared by compile, but has no subordinate or region. - { - continue; - } - for(k=0; kregion_ids); k++) - { - r2c_key.region_id=*((int*)utarray_eltptr(g2r->region_ids, k)); - r2c_key.vt_id=literal_id->vt_id; - HASH_FIND(hh, region2clause_hash, &r2c_key, sizeof(r2c_key), r2c_val); - if(!r2c_val) - { - r2c_val=ALLOC(struct region2clause_value, 1); - r2c_val->key=r2c_key; - r2c_val->group_id=g2r->group_id; - utarray_new(r2c_val->clause_ids, &ut_clause_id_icd); - HASH_ADD(hh, region2clause_hash, key, sizeof(r2c_val->key), r2c_val); - } - if(utarray_find(r2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id)) - { - continue; - } - utarray_push_back(r2c_val->clause_ids, &(clause_state->clause_id)); - utarray_sort(r2c_val->clause_ids, compare_clause_id); - } - } - } - - } - int tmp1=0, tmp2=0; - HASH_ITER(hh, g2r_hash, g2r, g2r_tmp) - { - HASH_DEL(g2r_hash, g2r); - //Sanity Check - utarray_sort(g2r->region_ids, compare_region_id); - for(i=1; iregion_ids); i++) - { - tmp1=*((int*)utarray_eltptr(g2r->region_ids, i-1)); - tmp2=*((int*)utarray_eltptr(g2r->region_ids, i)); - assert(tmp1!=tmp2); - } - utarray_free(g2r->region_ids); - g2r->region_ids=NULL; - free(g2r); - } - - MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, - "Build region2clause hash with %llu element.", - HASH_COUNT(region2clause_hash)); - - return region2clause_hash; -} -int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) -{ - int ret=0; - struct bool_matcher* new_bm=NULL, *old_bm=NULL; - struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL; - - //Read hier from update thread is OK. - if(!hier->changed_flag) - { - return ret; - } - ret=Maat_hierarchy_build_top_groups(hier); - new_bm=Maat_hierarchy_build_bool_matcher(hier); - new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier); - - pthread_rwlock_wrlock(&hier->rwlock); - - old_bm=hier->bm; - old_region2clause_hash=hier->hash_region2clause; - - hier->bm=new_bm; - hier->hash_region2clause=new_region2clause_hash; - hier->changed_flag=0; - pthread_rwlock_unlock(&hier->rwlock); - - Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); - Maat_garbage_bagging(hier->ref_garbage_bin, old_region2clause_hash, (void (*)(void*))Maat_hierarchy_free_region2clause_hash); - - return ret; -} - - -struct Maat_hierarchy_compile_mid -{ - int thread_num; - int Nth_scan; - time_t hier_ver; - size_t this_scan_region_hit_cnt; - int not_clause_hitted_flag; - int is_no_count_scan; - size_t hit_path_cnt; - - UT_array* _internal_hit_paths; - UT_array* _all_hit_clause_array; - UT_array* this_scan_hit_clause_ids; -}; - -struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hierarchy* hier, int thread_num) -{ - struct Maat_hierarchy_compile_mid* mid=ALLOC(struct Maat_hierarchy_compile_mid, 1); - mid->thread_num=thread_num; - mid->hier_ver=hier->version; - utarray_new(mid->_internal_hit_paths, &ut_hit_path_icd); - utarray_new(mid->_all_hit_clause_array, &ut_clause_id_icd); - utarray_new(mid->this_scan_hit_clause_ids, &ut_clause_id_icd); - return mid; -} -void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid) -{ - utarray_free(mid->_internal_hit_paths); - utarray_free(mid->_all_hit_clause_array); - utarray_free(mid->this_scan_hit_clause_ids); - free(mid); -} -int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid) -{ - return mid->not_clause_hitted_flag; -} - -static int Maat_hierarchy_compile_has_literal(struct Maat_hierarchy_compile* compile, struct Maat_hierarchy_literal_id* literal_id) -{ - int i=0; - struct Maat_hierarchy_literal_id* tmp=NULL; - struct Maat_hierarchy_clause_state* clause_state=NULL; - for(i=0; iclause_states+i; - if(!clause_state->in_use) - { - continue; - } - tmp=(struct Maat_hierarchy_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); - if(tmp) - { - assert(tmp->group_id==literal_id->group_id && tmp->vt_id==literal_id->vt_id); - return 1; - } - } - return 0; -} -static int Maat_hierarchy_is_hit_path_existed(const struct Maat_hit_path_t* hit_paths, size_t n_path, const struct Maat_hit_path_t* find) -{ - size_t i=0; - for(i=0; iexpr_match_buff+mid->thread_num*MAX_SCANNER_HIT_NUM; - struct Maat_hit_path_t tmp_path; - if(hier->version!=mid->hier_ver) - { - return 0; - } - pthread_rwlock_rdlock(&hier->rwlock); - - for(i=0; i_internal_hit_paths); i++) - { - p=(struct Maat_hierarchy_internal_hit_path*)utarray_eltptr(mid->_internal_hit_paths, i); - - HASH_FIND_INT(hier->hash_region_by_id, &(p->region_id), region); - if(!region) - { - continue; - } - group=region->ref_parent_group; - if(group->top_group_cnt==0 && n_made_by_regionNth_scan; - hit_paths[n_made_by_region].region_id=p->region_id; - hit_paths[n_made_by_region].sub_group_id=group->group_id; - hit_paths[n_made_by_region].top_group_id=-1; - hit_paths[n_made_by_region].virtual_table_id=p->virtual_table_id; - hit_paths[n_made_by_region].compile_id=-1; - n_made_by_region++; - } - else - { - for(j=0; jtop_group_cnt&& n_made_by_regionNth_scan; - hit_paths[n_made_by_region].region_id=p->region_id; - hit_paths[n_made_by_region].sub_group_id=group->group_id; - hit_paths[n_made_by_region].top_group_id=group->top_group_ids[j]; - hit_paths[n_made_by_region].virtual_table_id=p->virtual_table_id; - hit_paths[n_made_by_region].compile_id=-1; - } - } - } - - bool_match_ret=bool_matcher_match(hier->bm, - (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array), - expr_match, MAX_SCANNER_HIT_NUM); - for(i=0; imagic==MAAT_HIER_COMPILE_MAGIC); - assert((unsigned long long)compile->compile_id==expr_match[i].expr_id); - if(compile->actual_clause_num==0 || !compile->user_data) - { - continue; - } - for(j=0; jcompile_id; - } - else - { - tmp_path=hit_paths[j]; - tmp_path.compile_id=compile->compile_id; - if(Maat_hierarchy_is_hit_path_existed(hit_paths, n_made_by_region+n_made_by_compile, &tmp_path)) - { - hit_paths[n_made_by_region+n_made_by_compile]=tmp_path; - n_made_by_compile++; - } - } - } - } - } - - pthread_rwlock_unlock(&hier->rwlock); - return n_made_by_region+n_made_by_compile; -} - - -void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result) -{ - size_t i=0; - unsigned long long *clause_id=0; - - if(mid->Nth_scan!=Nth_scan) - { - assert(mid->this_scan_region_hit_cnt==0); - mid->Nth_scan=Nth_scan; - utarray_clear(mid->this_scan_hit_clause_ids); - } - - int ret=0; - ret=Maat_hierarchy_hit_path_add(mid->_internal_hit_paths, region_id, virtual_table_id, Nth_scan, Nth_region_result); - if(!ret) - { - return; - } - mid->hit_path_cnt++; - mid->this_scan_region_hit_cnt++; - - - struct region2clause_value* r2c_val=NULL; - struct region2clause_key r2c_key; - r2c_key.region_id=region_id; - r2c_key.vt_id=virtual_table_id; - HASH_FIND(hh, hier->hash_region2clause, &r2c_key, sizeof(r2c_key), r2c_val); - if(!r2c_val) - { - return; - } - size_t new_clause_idx=utarray_len(mid->this_scan_hit_clause_ids); - for(i=0; iclause_ids); i++) - { - clause_id=(unsigned long long*)utarray_eltptr(r2c_val->clause_ids, i); - if(utarray_find(mid->_all_hit_clause_array, clause_id, compare_clause_id)) - { - continue; - } - utarray_push_back(mid->this_scan_hit_clause_ids, clause_id); - } - if(utarray_len(mid->this_scan_hit_clause_ids)-new_clause_idx) - { - utarray_reserve(mid->_all_hit_clause_array, utarray_len(mid->this_scan_hit_clause_ids)-new_clause_idx); - for(i=new_clause_idx; ithis_scan_hit_clause_ids); i++) - { - clause_id=(unsigned long long *)utarray_eltptr(mid->this_scan_hit_clause_ids, i); - utarray_push_back(mid->_all_hit_clause_array, clause_id); - } - utarray_sort(mid->_all_hit_clause_array, compare_clause_id); - } - return; -} -static int Maat_hierarchy_compile_has_clause(struct Maat_hierarchy_compile* compile, unsigned long long clause_id) -{ - size_t i=0; - struct Maat_hierarchy_clause_state* clause_state=NULL; - for(i=0; iclause_states+i; - if(!clause_state->in_use) - { - continue; - } - if(clause_state->clause_id==clause_id) - { - return 1; - } - } - return 0; -} - - -static size_t Maat_hierarchy_compile_mid_if_new_hit_compile(struct Maat_hierarchy_compile_mid* mid, struct Maat_hierarchy_compile* compile) -{ - size_t r_in_c_cnt=0, i=0; - int ret=0; - - unsigned long long new_hit_clause_id=0; - for(i=0; ithis_scan_hit_clause_ids); i++) - { - - new_hit_clause_id=*(unsigned long long*)utarray_eltptr(mid->this_scan_hit_clause_ids, i); - ret=Maat_hierarchy_compile_has_clause(compile, new_hit_clause_id); - if(ret) - { - r_in_c_cnt++; - } - } - return r_in_c_cnt; -} - - -int Maat_hierarchy_region_compile(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz) -{ - int bool_match_ret=0, i=0; - struct Maat_hierarchy_compile* compile=NULL; - struct bool_expr_match *expr_match=hier->expr_match_buff+mid->thread_num*MAX_SCANNER_HIT_NUM; - - size_t r_in_c_cnt=0, this_scan_region_hits=mid->this_scan_region_hit_cnt; - size_t ud_result_cnt=0; - if(!hier->bm||0==utarray_len(mid->_all_hit_clause_array)||hier->version!=mid->hier_ver) - { - mid->this_scan_region_hit_cnt=0; - return 0; - } - bool_match_ret=bool_matcher_match(hier->bm, - (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array), - expr_match, MAX_SCANNER_HIT_NUM); - for(i=0; imagic==MAAT_HIER_COMPILE_MAGIC); - assert((unsigned long long)compile->compile_id==expr_match[i].expr_id); - if(compile->actual_clause_num==0) - { - continue; - } - r_in_c_cnt=Maat_hierarchy_compile_mid_if_new_hit_compile(mid, compile); - if(compile->not_clause_cnt>0 && !is_last_compile) - { - mid->not_clause_hitted_flag=1; - } - else if(compile->user_data)//For compile may be dettached by Maat_hierarchy_compile_dettach_user_data, only return non-NULL userdata. - { - if(r_in_c_cnt>0 || //compile hitted becasue of new reigon - this_scan_region_hits==0) //or hit a compile that refer a NOT-logic group in previous scan. - { - user_data_array[ud_result_cnt]=compile->user_data; - ud_result_cnt++; - } - } - } - - mid->this_scan_region_hit_cnt=0; - return ud_result_cnt; -} - diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp deleted file mode 100644 index efadc00..0000000 --- a/src/entry/Maat_rule.cpp +++ /dev/null @@ -1,2815 +0,0 @@ -#include -#include -#include //tolower -#include -#include //inet_pton -#include //inet_pton -#include //inet_pton -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "Maat_rule.h" -#include "Maat_rule_internal.h" -#include "Maat_utils.h" -#include "Maat_hierarchy.h" -#include "Maat_garbage_collection.h" - -#include "json2iris.h" -#include "cJSON.h" -#include "dynamic_array.h" -#include "alignment_int64.h" -#include "config_monitor.h" - -#include "map_str2int.h" -#include "stream_fuzzy_hash.h" -#include "gram_index_engine.h" - -#include "bool_matcher.h" - -#define GIT_VERSION_CATTER(v) __attribute__((__used__)) const char * GIT_VERSION_##v = NULL -#define GIT_VERSION_EXPEND(v) GIT_VERSION_CATTER(v) - -#ifdef __cplusplus -extern "C" -{ -#endif - - /* VERSION TAG */ -#ifdef GIT_VERSION - GIT_VERSION_EXPEND(GIT_VERSION); -#else - static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL; -#endif -#undef GIT_VERSION_CATTER -#undef GIT_VERSION_EXPEND - -#ifdef __cplusplus -} -#endif - -int MAAT_FRAME_VERSION_3_7_0_20220823=1; - -int is_valid_table_name(const char* str) -{ - size_t i=0, integer_cnt=0; - for(i=0; i='0'&&str[i]<='9') - { - integer_cnt++; - } - } - if(strlen(str)==0 || - integer_cnt==strlen(str) || - 0==strcasecmp(str, "null")) - { - return 0; - } - return 1; -} -int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) -{ - switch(expr_type) - { - case EXPR_TYPE_STRING: - case EXPR_TYPE_AND: - case EXPR_TYPE_REGEX: - case EXPR_TYPE_OFFSET: - return 1; - default: - return 0; - } -} -int is_valid_match_method(enum MAAT_MATCH_METHOD match_method) -{ - switch(match_method) - { - case MATCH_METHOD_SUB: - case MATCH_METHOD_SUFFIX: - case MATCH_METHOD_PREFIX: - case MATCH_METHOD_COMPLETE: - return 1; - default: - return 0; - } -} - - -iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) -{ - const char *from_s=charset_get_name(from); - const char *to_s=charset_get_name(to); - iconv_t cd; - if(from==CHARSET_GBK&&to==CHARSET_BIG5) - { - from_s="gb2312"; - } - if(from>=MAX_CHARSET_NUM||to>=MAX_CHARSET_NUM) - { - return (iconv_t)-1; - } - - if(scanner->iconv_handle[to][from]==NULL) - { - scanner->iconv_handle[to][from]=iconv_open(to_s, from_s); - } - cd=scanner->iconv_handle[to][from]; - return cd; -} - -int iconv_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) -{ - size_t ret; - int copy_len=0; - char* copy_buf=NULL; - if(srclen==0||src==NULL) - { - return -1; - } - iconv_t cd=maat_iconv_open(scanner,to, from); - if(cd!=(iconv_t)-1) - { - char * pInBuff=src; - size_t iInBuffLen=srclen; - - size_t iOutBuffLen=10*iInBuffLen; - char * pOutBuff=(char *)malloc(iOutBuffLen); - - char * pLeftBuff=pOutBuff; - size_t iLeftLen=iOutBuffLen; - - ret=iconv(cd, &pInBuff, &iInBuffLen, &pLeftBuff, &iLeftLen); - if(ret!=(size_t)(-1)) - { - - if(to==CHARSET_UNICODE&& - (*(unsigned short*)pOutBuff==0xFFFE||*(unsigned short*)pOutBuff==0XFEFF))//jump unicode 2 bytes BOM, 0xFF 0xFE - { - copy_len=iOutBuffLen-iLeftLen-2; - copy_buf=pOutBuff+2; - } - else - { - copy_len=iOutBuffLen-iLeftLen; - copy_buf=pOutBuff; - } - assert(copy_len<=*dstlen); - *dstlen=copy_len; - memcpy(dst,copy_buf,*dstlen); - - free(pOutBuff); - return 1; - } - else - { - free(pOutBuff); - return -1; - } - } - else - { - return -1; - } - -} -int URLEncode(const char* str, const int strSize, char* result, const int resultSize) -{ - int i; - int j = 0;//for result index - char ch; - - if ((str==NULL) || (result==NULL) || (strSize<=0) || (resultSize<=0)) - { - return -1; - } - - for ( i=0; (i='A') && (ch<='Z')) || - ((ch>='a') && (ch<='z')) || - ((ch>='0') && (ch<='9'))) - { - result[j++] = ch; - } - else if (ch == ' ') - { - result[j++] = '+'; - } - else if (ch == '.' || ch == '-' || ch == '_' || ch == '*') - { - result[j++] = ch; - } - else - { - if (j+3 < resultSize) - { - sprintf(result+j, "%%%02X", (unsigned char)ch); - j += 3; - } - else - { - return -1; - } - } - } - - result[j] = '\0'; - return j; -} -int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, const int dstsize) -{ - int i=0,j=0; - assert(srclen%2==0);//unicode must be 2 bytes aligned. - while(i0;p[i]=p[i]/2) - { - bits_cnt++; - } - } - return bits_cnt; -} -//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/æœé˜³/åŽä¸¥åŒ—里/甲22å·},{"tag":"isp","value":"电信"}]} -int parse_accept_tag(const char* value, struct rule_tag** result, void* logger) -{ - cJSON* json=NULL, *array=NULL,*tag=NULL, *tmp=NULL; - struct rule_tag* p=NULL; - int n_tags=0; - json=cJSON_Parse(value); - if(!json) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "MAAT_OPT_ACCEPT_TAGS Error before: %-200.200s",cJSON_GetErrorPtr()); - return 0; - } - array=cJSON_GetObjectItem(json, "tags"); - n_tags=cJSON_GetArraySize(array); - p=ALLOC(struct rule_tag, n_tags); - for(int i=0;ivaluestring); - tmp=cJSON_GetObjectItem(tag, "value"); - p[i].tag_val=_maat_strdup(tmp->valuestring); - } - cJSON_Delete(json); - *result=p; - return n_tags; -} -static int compare_each_tag(cJSON* tag_obj, const struct rule_tag* accept_tags, int n_accept) -{ - const char* tag_name; - const char* tag_val; - int n_val; - cJSON *tab_name_obj=NULL, *tag_vals_array=NULL, *tag_val_obj=NULL; - - int i=0, j=0, name_matched=0; - tab_name_obj=cJSON_GetObjectItem(tag_obj,"tag"); - if(!tab_name_obj||tab_name_obj->type!=cJSON_String) - { - goto error_out; - } - tag_name=tab_name_obj->valuestring; - tag_vals_array=cJSON_GetObjectItem(tag_obj,"value"); - if(!tag_vals_array||tag_vals_array->type!=cJSON_Array) - { - goto error_out; - } - n_val=cJSON_GetArraySize(tag_vals_array); - for(i=0;itype!=cJSON_String) - { - goto error_out; - } - tag_val=tag_val_obj->valuestring; - // compare a/b/c with a/b/c/d is a miss. - if(strlen(accept_tags[i].tag_val)0) - { - return 0; - } - else - { - return 1; - } -error_out: - return -1; - -} -//@param tag_set likes [{"tag":"location","value":["北京/æœé˜³/åŽä¸¥åŒ—里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] -static int compare_each_tag_set(cJSON* tag_set, const struct rule_tag* accept_tags, int n_accept) -{ - cJSON *tag_obj=NULL; - int n_tag=0, ret=0, matched=0; - n_tag=cJSON_GetArraySize(tag_set); - for(int i=0; itype!=cJSON_Object) - { - goto error_out; - } - ret=compare_each_tag(tag_obj, accept_tags, n_accept); - if(ret<0) - { - return -1; - } - if(ret==1) - { - matched++; - } - } - if(matched==n_tag) - { - return 1; - } - else - { - return 0; - } -error_out: - return -1; -} -//@param value {"tag_sets":[[{"tag":"location","value":["北京/æœé˜³/åŽä¸¥åŒ—里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["è”通"]}]]} -//@return 1 on match, 0 on not match, -1 on error. -static int compare_accept_tag(const char* value,const struct rule_tag* accept_tags, int n_tags) -{ - cJSON *json=NULL; - cJSON *tag_set_array=NULL, *tag_set=NULL; - int ret=-1, n_set=0; - json=cJSON_Parse(value); - if(!json) - { - goto error_out; - } - tag_set_array=cJSON_GetObjectItem(json, "tag_sets"); - if(!tag_set_array||tag_set_array->type!=cJSON_Array) - { - goto error_out; - } - n_set=cJSON_GetArraySize(tag_set_array); - for(int i=0; itype!=cJSON_Array) - { - goto error_out; - } - ret=compare_each_tag_set(tag_set, accept_tags, n_tags); - if(ret!=0)//match or error occurs. - { - break; - } - } -error_out: - cJSON_Delete(json); - return ret; -} - -MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc) -{ - MAAT_RULE_EX_DATA ad=NULL; - struct Maat_rule_t rule; - fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp); - return ad; -} -void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc) -{ - struct Maat_rule_t rule; - fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp); - return; -} - -void EMPTY_FREE(void*p) -{ - return; -} - -struct Maat_compile_rule* create_compile_rule(struct Maat_rule_head* p_head, const char* service_define, int declared_grp_num, double exec_seq, const struct Maat_table_schema* table) -{ - int i=0; - struct Maat_compile_rule*p=ALLOC(struct Maat_compile_rule, 1); - p->magic_num=COMPILE_RULE_MAGIC; - p->head=*p_head; - p->declared_clause_num=declared_grp_num; - p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM); - - //protect by feather->background_update_mutex - p->ref_table=table; - p->head.serv_def_len=strlen(service_define)+1; - p->service_defined=ALLOC(char, p->head.serv_def_len); - memcpy(p->service_defined, service_define, p->head.serv_def_len); - p->evaluation_order=exec_seq; - - for(i=0; icompile.ex_data_num; i++) - { - p->ads[i]=rule_ex_data_new(&p->head, p->service_defined, table->compile.ex_desc+i); - } - p->is_valid=1; - p->compile_id=p_head->config_id; - pthread_rwlock_init(&p->rwlock, NULL); - return p; -} - -void destroy_compile_rule(struct Maat_compile_rule* compile_rule) -{ - int i=0; - const struct compile_table_schema* compile_desc= &(compile_rule->ref_table->compile); - assert(compile_rule->magic_num==COMPILE_RULE_MAGIC); - for(i=0; iex_data_num; i++) - { - rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i); - compile_rule->ads[i]=NULL; - } - free(compile_rule->ads); - - compile_rule->is_valid=0; - compile_rule->declared_clause_num=-1; - free(compile_rule->service_defined); - compile_rule->service_defined=NULL; - free(compile_rule); - return; -} - - -scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset) -{ - scan_rule_t* p_rule=(scan_rule_t* )calloc(sizeof(scan_rule_t),1); - p_rule->rule_type=RULETYPE_STR; - p_rule->sub_type=sub_type; - - p_rule->string_rule.case_sensitive=is_case_sensitive; - p_rule->string_rule.match_mode=0; - p_rule->string_rule.l_offset=-1; - p_rule->string_rule.r_offset=-1; - switch(match_method) - { - case MATCH_METHOD_COMPLETE: - p_rule->string_rule.match_mode=1; - break; - case MATCH_METHOD_PREFIX: - p_rule->string_rule.l_offset=-2; - break; - case MATCH_METHOD_SUFFIX: - p_rule->string_rule.r_offset=-2; - break; - case MATCH_METHOD_SUB: - p_rule->string_rule.l_offset=l_offset; - p_rule->string_rule.r_offset=r_offset; - break; - default: - assert(0); - break; - } - p_rule->string_rule.len=len; - p_rule->string_rule.str=(char*)calloc(sizeof(char),len); - memcpy(p_rule->string_rule.str,string,len); - return p_rule; - -} -void destroy_rs_str_rule(scan_rule_t* p_rule) -{ - free(p_rule->string_rule.str); - free(p_rule); -} -scan_rule_t* create_rs_ip_rule(unsigned int sub_type,struct db_ip_rule_t *db_ip_rule) -{ - scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1); - if(db_ip_rule->addr_type==4) - { - p_rule->rule_type=RULETYPE_IPv4; - memcpy(&(p_rule->ipv4_rule),&(db_ip_rule->ipv4_rule),sizeof(p_rule->ipv4_rule)); - } - else - { - p_rule->rule_type=RULETYPE_IPv6; - memcpy(&(p_rule->ipv6_rule),&(db_ip_rule->ipv6_rule),sizeof(p_rule->ipv6_rule)); - } - p_rule->sub_type=sub_type; - return p_rule; -} -void destroy_rs_ip_rule(scan_rule_t* p) -{ - free(p); -} -scan_rule_t* create_rs_intval_rule(unsigned int sub_type,struct db_interval_rule *intval_rule) -{ - scan_rule_t *p_rule=(scan_rule_t*)calloc(sizeof(scan_rule_t),1); - p_rule->rule_type=RULETYPE_INT; - p_rule->sub_type=sub_type; - p_rule->interval_rule.lb=intval_rule->intval.lb; - p_rule->interval_rule.ub=intval_rule->intval.ub; - return p_rule; -} -void destroy_rs_intval_rule(scan_rule_t* p) -{ - free(p); -} - -struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para,int table_id) -{ - struct op_expr_t* op_expr=NULL; - op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1); - op_expr->no_effect_convert_cnt=0; - op_expr->convert_failed=0; - op_expr->p_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),1); - op_expr->p_expr->expr_id=expr_id; - op_expr->p_expr->operation=operation; - op_expr->p_expr->rnum=0; - op_expr->p_expr->rules=NULL; - op_expr->p_expr->tag=u_para; - op_expr->table_id=table_id; - return op_expr; -} -void destroy_op_expr(struct op_expr_t* op_expr) -{ - unsigned int i=0; - for(i=0;ip_expr->rnum;i++) - { - switch(op_expr->p_rules[i]->rule_type) - { - case RULETYPE_STR: - case RULETYPE_REG: - destroy_rs_str_rule(op_expr->p_rules[i]); - break; - case RULETYPE_IPv4: - case RULETYPE_IPv6: - destroy_rs_ip_rule(op_expr->p_rules[i]); - break; - case RULETYPE_INT: - destroy_rs_intval_rule(op_expr->p_rules[i]); - break; - default: - assert(0); - break; - } - op_expr->p_rules[i]=NULL; - } - free(op_expr->p_expr); - op_expr->p_expr=NULL; - free(op_expr); -} -void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) -{ - int idx=op_expr->p_expr->rnum; - op_expr->p_rules[idx]=p_rule; - op_expr->p_expr->rnum++; - op_expr->rule_type=p_rule->rule_type; - return; -} -void Maat_region_inner_free(struct Maat_region_inner* region) -{ - assert(region->magic_num==REGION_RULE_MAGIC); - assert(region->expr_id_cnt==0||region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb+1); - region->magic_num=0; - free(region); -} - -void Maat_region_inner_cancel_last_expr_id(struct Maat_region_inner* region) -{ - assert(region->expr_id_cnt==region->expr_id_ub-region->expr_id_lb+1); - region->expr_id_ub--; - region->expr_id_cnt--; - return; -} - -struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *feather) -{ - int scan_thread_num=feather->scan_thread_num; - UNUSED int ret=0; - - struct Maat_scanner* scanner=NULL; - scanner=ALLOC(struct Maat_scanner, 1); - scanner->hier=Maat_hierarchy_new(scan_thread_num, feather->logger, feather->garbage_bin); - Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, (void (*)(void*))destroy_compile_rule); - Maat_hierarchy_set_region_user_data_free_func(scanner->hier, (void (*)(void*))Maat_region_inner_free); - - scanner->district_map=maat_kv_store_new(); - - scanner->version=version; - scanner->cfg_num=0; - scanner->dedup_expr_num=0; - scanner->max_thread_num=scan_thread_num; - //optimized for CPU cache_alignment 64 - scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num); - scanner->region_update_q=MESA_lqueue_create(0, 0); - scanner->region=rulescan_initialize(scan_thread_num); - - //For best scan performance: - //1.Do NOT set this option, rulescan return no hit detail as default; - //2.Set necessary STR rule to QUICK; - if(feather->rule_scan_type==1) - { - rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0); - } - else if(feather->rule_scan_type==2) - { - rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0); - rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0); - } - scanner->ref_garbage_bin=feather->garbage_bin; - scanner->logger_ref=feather->logger; - scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); - scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); - scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num, feather->garbage_bin); - scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr); - return scanner; -} - - - -void destroy_maat_scanner(struct Maat_scanner*scanner) -{ - long q_cnt=0,data_size=0; - int i=0,j=0; - UNUSED int q_ret=0; - struct op_expr_t* op_expr=NULL; - if(scanner==NULL) - { - return; - } - rulescan_destroy(scanner->region); - - maat_kv_store_free(scanner->district_map); - scanner->district_map=NULL; - assert(scanner->tmp_district_map==NULL); - - q_cnt=MESA_lqueue_get_count(scanner->region_update_q); - for(i=0;iregion_update_q,&op_expr,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_op_expr(op_expr); - } - MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL); - free(scanner->region_rslt_buff); - scanner->region_rslt_buff=NULL; - free(scanner->gie_rslt_buff); - scanner->gie_rslt_buff=NULL; - alignment_int64_array_free(scanner->ref_cnt); - scanner->ref_cnt=NULL; - for(i=0;iiconv_handle[i][j]!=NULL) - { - iconv_close(scanner->iconv_handle[i][j]); - } - } - } - Maat_table_rt_manager_destroy(scanner->table_rt_mgr); - scanner->table_rt_mgr=NULL; - Maat_hierarchy_free(scanner->hier); - scanner->hier=NULL; - free(scanner); - return; -} - - -unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge) -{ - unsigned int sub_type=0; - if(do_charset_merge==TRUE) - { - sub_type=table_id<<4|CHARSET_NONE; - } - else - { - sub_type=table_id<<4|charset; - } - assert(sub_typerules); - free(p); - return; -} -struct _region_stat_t -{ - int cfg_num; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; -}; -void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size) -{ - int op=0; - if(op_expr->p_expr->operation==0)//add - { - op=1; - } - else if(op_expr->p_expr->operation==1)//delete - { - op=-1; - } - else - { - assert(0); - } - region_stat[op_expr->table_id].cfg_num+=op; - switch(op_expr->rule_type) - { - case RULETYPE_STR: - region_stat[op_expr->table_id].expr_rule_cnt+=op; - break; - case RULETYPE_REG: - region_stat[op_expr->table_id].regex_rule_cnt+=op; - break; - case RULETYPE_INT: - break; - case RULETYPE_IPv4: - region_stat[op_expr->table_id].ipv4_rule_cnt+=op; - break; - case RULETYPE_IPv6: - region_stat[op_expr->table_id].ipv6_rule_cnt+=op; - break; - default: - assert(0); - break; - } - return; -} - -void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner) -{ - long data_size=0, i=0; - unsigned int j=0; - int ret=0; - unsigned int failed_ids[MAX_FAILED_NUM]; - char failed_info[512], *p=NULL; - UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; - memset(failed_ids,0,sizeof(failed_ids)); - memset(failed_info,0,sizeof(failed_info)); - const long q_cnt=MESA_lqueue_get_count(expr_queue); - struct timespec start,end; - unsigned long long update_interval=0; - size_t max_table_num=maat_scanner->max_table_num; - struct _region_stat_t region_counter[max_table_num]; - memset(region_counter, 0, sizeof(region_counter)); - struct Maat_table_runtime* table_rt=NULL; - if(q_cnt==0) - { - return; - } - boolean_expr_t* to_update_expr= ALLOC(boolean_expr_t, q_cnt); - struct op_expr_t* op_expr=NULL; - for(i=0;ip_expr,sizeof(boolean_expr_t)); - //make a whole memory chunk - to_update_expr[i].rules= ALLOC(scan_rule_t, op_expr->p_expr->rnum); - for(j=0;jp_expr->rnum;j++) - { - memcpy(&(to_update_expr[i].rules[j]),op_expr->p_rules[j],sizeof(scan_rule_t)); - if(to_update_expr[i].rules[j].rule_type==RULETYPE_REG||to_update_expr[i].rules[j].rule_type==RULETYPE_STR) - { - to_update_expr[i].rules[j].string_rule.str=(char*)calloc(sizeof(char),to_update_expr[i].rules[j].string_rule.len); - memcpy(to_update_expr[i].rules[j].string_rule.str - ,op_expr->p_rules[j]->string_rule.str - ,to_update_expr[i].rules[j].string_rule.len); - } - } - - count_rs_region(op_expr, region_counter, max_table_num); - destroy_op_expr(op_expr); - op_expr=NULL; - } - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module , - "rs_handle %p rulescan_update %ld rules.", rs_handle, q_cnt); - clock_gettime(CLOCK_MONOTONIC, &start); - ret=rulescan_update(rs_handle, to_update_expr,q_cnt, failed_ids, MAX_FAILED_NUM); - clock_gettime(CLOCK_MONOTONIC, &end); - if(ret!=1) - { - p=failed_info; - for(i=0;i10;i++) - { - p+=snprintf(p,sizeof(failed_info)-(p-failed_info),"%d,",failed_ids[i+1]); - } - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "rulescan_update error,when batch update %ld rules,regex error %u.",q_cnt,failed_ids[0]); - assert(0); - } - update_interval=(end.tv_sec-start.tv_sec)*1000000000+end.tv_nsec-start.tv_nsec; - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "rs_handle %p rulescan_update with %2.2e (%llu) ns." - ,rs_handle - ,(double)update_interval - ,update_interval); - //update scanner's region cnt; - for(i=0; (size_t)itable_rt_mgr, i); - if(table_rt==NULL) - { - continue; - } - switch(table_rt->table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; - table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; - assert(table_rt->expr.expr_rule_cnt>=0); - assert(table_rt->expr.regex_rule_cnt>=0); - break; - case TABLE_TYPE_IP: - case TABLE_TYPE_IP_PLUS: - table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; - table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; - break; - default: - break; - } - assert(table_rt->origin_rule_num>=0); - } - for(i=0;idistrict_map, district_str, &district_id); - if(map_ret<0) - { - if(scanner->tmp_district_map==NULL) - { - scanner->tmp_district_map=maat_kv_store_duplicate(scanner->district_map); - } - map_ret=maat_kv_read(scanner->tmp_district_map, district_str, &district_id); - if(map_ret<0) - { - district_id=scanner->district_num; - maat_kv_register(scanner->tmp_district_map, district_str, district_id); - scanner->district_num++; - } - } - return district_id; -} -struct Maat_region_inner* Maat_region_inner_new(int group_id, int region_id, int table_id, int district_id) -{ - struct Maat_region_inner* region=ALLOC(struct Maat_region_inner, 1); - region->magic_num=REGION_RULE_MAGIC; - region->region_id=region_id; - region->group_id=group_id; - region->table_id=table_id; - region->district_id=district_id; - return region; -} -void Maat_region_inner_add_expr_id(struct Maat_region_inner* region, int expr_id) -{ - if(region->expr_id_cnt==0) - { - region->expr_id_lb=region->expr_id_ub=expr_id; - } - else - { - assert(region->expr_id_ub+1==expr_id); - region->expr_id_ub=expr_id; - } - region->expr_id_cnt++; - return; -} - -int add_expr_ng_rule(struct Maat_table_schema* table, struct db_expr_rule_t* db_rule, struct Maat_scanner *scanner, void* logger) -{ - int district_id=-1; - if(table->table_type==TABLE_TYPE_EXPR_PLUS) - { - assert(strlen(db_rule->district)>0); - str_unescape(db_rule->district); - district_id=get_district_id(scanner, db_rule->district); - } - struct hs_expression *hs_expr=ALLOC(struct hs_expression, 1); - struct hs_pattern *sub_pattern=NULL; - int i=0, j=0, ret=0; - int sub_expr_cnt=0; - char *p=NULL, *saveptr=NULL, *region_string=NULL; - char *tmp=NULL, *tmp_pattern=NULL; - hs_expr->type=db_rule->expr_type; - struct Maat_region_inner* user_tag=NULL; - - switch(db_rule->expr_type) - { - case EXPR_TYPE_AND: - case EXPR_TYPE_OFFSET: - for(i=0, p=db_rule->keywords; ; i++,p=NULL) - { - tmp=strtok_r_esc(p, '&', &saveptr); - if(tmp==NULL) - { - break; - } - if(i>=MAAT_MAX_EXPR_ITEM_NUM) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id); - goto error_out; - } - if(strlen(tmp)==0) continue; - sub_pattern=hs_expr->sub_patterns[hs_expr->n_sub_pattern]; - tmp_pattern=tmp; - if(db_rule->expr_type==EXPR_TYPE_OFFSET) - { - sscanf(tmp, "%d-%d:", &(sub_pattern->start_offset),&(sub_pattern->end_offset)); - tmp_pattern=(char*)memchr(tmp_pattern, ':', strlen(tmp_pattern)); - if(tmp_pattern==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d invalid offset keyword format.", table->table_name[table->updating_name], db_rule->region_id); - goto error_out; - } - tmp_pattern++;//skip ':' - } - else - { - sub_pattern->start_offset=sub_pattern->end_offset=-1; - } - sub_pattern->pattern=_maat_strdup(tmp_pattern); - str_unescape(hs_expr->sub_patterns[i].pattern); - hs_expr->n_sub_pattern++; - } - break; - case EXPR_TYPE_REGEX: - case EXPR_TYPE_STRING: - hs_expr->n_sub_pattern=1; - switch(db_rule->match_method) - { - case MATCH_METHOD_COMPLETE: - hs_expr->sub_patterns[0].start_offset=-2; - hs_expr->sub_patterns[0].end_offset=-2; - break; - case MATCH_METHOD_PREFIX: - hs_expr->sub_patterns[0].start_offset=-2; - hs_expr->sub_patterns[0].end_offset=-1; - break; - case MATCH_METHOD_SUFFIX: - hs_expr->sub_patterns[0].start_offset=-1; - hs_expr->sub_patterns[0].end_offset=-2; - break; - case MATCH_METHOD_SUB: - hs_expr->sub_patterns[0].start_offset=-1; - hs_expr->sub_patterns[0].end_offset=-1; - break; - default: - assert(0); - break; - } - hs_expr->sub_patterns[0].pattern=_maat_strdup(tmp); - str_unescape(hs_expr->sub_patterns[0].pattern); - break; - default: - break; - } - user_tag=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, district_id); - ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, user_tag); - if(ret!=0) - { - goto error_out; - } - hs_expr->user_tag=user_tag; - for(i=0; in_sub_pattern; i++) - { - if(db_rule->is_hexbin) - { - hex2bin(hs_expr->sub_patterns[i].pattern, int hex_len, char * binary, int size) - } - } -error_out: - - free(hs_expr); - if(user_tag) Maat_region_inner_free(user_tag); - return -1; -} -int add_expr_rule(struct Maat_table_schema* table,struct db_expr_rule_t* db_rule,struct Maat_scanner *scanner,void* logger) -{ - unsigned int i=0,j=0; - char* p=NULL,*saveptr=NULL,*region_string=NULL; - int region_str_len=0,ret=0,k=0; - int expr_id=0,district_id=-1; - struct string_table_schema* expr_desc=&(table->expr); - scan_rule_t*p_rule=NULL; - - enum MAAT_CHARSET dst_charset=CHARSET_NONE; - char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM], *tmp=NULL; - int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; - for(i=0;itable_type==TABLE_TYPE_EXPR_PLUS) - { - assert(strlen(db_rule->district)>0); - str_unescape(db_rule->district); - district_id=get_district_id(scanner, db_rule->district); - } - - - switch(db_rule->expr_type) - { - case EXPR_TYPE_AND: - case EXPR_TYPE_REGEX: - for(i=0,p=db_rule->keywords;;i++,p=NULL) - { - tmp=strtok_r_esc(p, '&', &saveptr); - if(tmp==NULL) - { - break; - } - if(i>=MAAT_MAX_EXPR_ITEM_NUM) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id); - return -1; - } - sub_key_array[i]=tmp; - if(db_rule->expr_type==EXPR_TYPE_REGEX) - { - sub_key_array[i]=str_unescape_and(sub_key_array[i]);//regex should use str_unescape_and - } - else - { - sub_key_array[i]=str_unescape(sub_key_array[i]); - } - } - sub_expr_cnt=i; - break; - case EXPR_TYPE_OFFSET: - for(i=0,p=db_rule->keywords;;i++,p=NULL) - { - tmp=strtok_r_esc(p, '&', &saveptr); - if(tmp==NULL) - { - break; - } - if(i>=MAAT_MAX_EXPR_ITEM_NUM) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d too many expr.", table->table_name[table->updating_name], db_rule->region_id); - return -1; - } - sub_key_array[i]=tmp; - sscanf(sub_key_array[i], "%d-%d:", &(key_left_offset[i]),&(key_right_offset[i])); - if(!(key_left_offset[i]>=0&&key_right_offset[i]>0&&key_left_offset[i]<=key_right_offset[i])) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d invalid offset.", table->table_name[table->updating_name], db_rule->region_id); - return -1; - } - sub_key_array[i]=(char*)memchr(sub_key_array[i], ':', strlen(sub_key_array[i])); - if(sub_key_array[i]==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d invalid offset keyword format.", table->table_name[table->updating_name], db_rule->region_id); - return -1; - } - sub_key_array[i]++;//jump over ':' - sub_key_array[i]=str_unescape(sub_key_array[i]); - } - sub_expr_cnt=i; - break; - case EXPR_TYPE_STRING: - sub_expr_cnt=1; - sub_key_array[0]=db_rule->keywords; - sub_key_array[0]=str_unescape(sub_key_array[0]); - break; - default: - break; - } - for(k=0;ktable_name[table->updating_name],db_rule->region_id); - //this sub string will be skipped before iconv_convert - } - } - - u_para=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, district_id); - ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, u_para); - if(ret!=0) - { - Maat_region_inner_free(u_para); - u_para=NULL; - return -1; - } - - if(db_rule->is_hexbin==FALSE && db_rule->expr_type!=EXPR_TYPE_REGEX) - { - for(j=0;jdst_charset[j]; - if(dst_charset==CHARSET_NONE) - { - break; - } - expr_id=scanner->exprid_generator++; - Maat_region_inner_add_expr_id(u_para, expr_id); - op_expr=create_op_expr(expr_id, - RULESCAN_OP_ADD, - u_para, - table->table_id); - - for(k=0;ksrc_charset!=dst_charset)//need convert - { - - ret=universal_charset_convert(scanner,expr_desc->src_charset, dst_charset, - sub_key_array[k],strlen(sub_key_array[k]), - region_string, ®ion_str_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module , - "Table %s region cfg %d charset convert from %s to %s failed.", - table->table_name, - db_rule->region_id, - charset_get_name(expr_desc->src_charset), - charset_get_name(dst_charset)); - free(region_string); - op_expr->convert_failed++; - expr_desc->iconv_err_cnt++; - break; - } - if(region_str_len==(int)strlen(sub_key_array[k])&& - 0==memcmp(sub_key_array[k],region_string,region_str_len)) - { - op_expr->no_effect_convert_cnt++; - } - } - else - { - memcpy(region_string,sub_key_array[k],strlen(sub_key_array[k])); - region_str_len=strlen(sub_key_array[k]); - } - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge) - ,db_rule->match_method - ,db_rule->is_case_sensitive - ,region_string - ,region_str_len - ,key_left_offset[k] - ,key_right_offset[k]); - op_expr_add_rule(op_expr, p_rule); - free(region_string); - region_string=NULL; - } - //if each sub string's convert take no effect and src charset is one of the dst. - //if any sub expr convert failed - if((TRUE==expr_desc->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)|| - op_expr->convert_failed>0) - { - scanner->dedup_expr_num++; - Maat_region_inner_cancel_last_expr_id(u_para); - destroy_op_expr(op_expr); - //redeem expr_id - scanner->exprid_generator--; - op_expr=NULL; - } - else - { - MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); - } - } - - } - else //For hexbin and regex, no need to do charset conversion. - { - expr_id=scanner->exprid_generator++; - Maat_region_inner_add_expr_id(u_para, expr_id); - op_expr=create_op_expr(expr_id, - 0, //add - u_para, - table->table_id - ); - for(k=0;kexpr_type==EXPR_TYPE_REGEX) - { - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge), - db_rule->match_method, - db_rule->is_case_sensitive, - sub_key_array[k], - strlen(sub_key_array[k]), - key_left_offset[k], - key_right_offset[k]); - p_rule->rule_type=RULETYPE_REG; - } - else - { - region_str_len=strlen(sub_key_array[k])+1; - region_string=ALLOC(char, region_str_len); - region_str_len=hex2bin(sub_key_array[k], strlen(sub_key_array[k]), region_string, region_str_len); - - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge), - db_rule->match_method, - db_rule->is_case_sensitive, - region_string, - region_str_len, - key_left_offset[k], - key_right_offset[k]); - - free(region_string); - region_string=NULL; - } - op_expr_add_rule(op_expr, p_rule); - } - MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); - } - return 0; -} -int add_ip_rule(struct Maat_table_schema* table, struct db_ip_rule_t* db_ip_rule, struct Maat_scanner *scanner, void* logger) -{ - scan_rule_t* p_rule=NULL; - struct op_expr_t* op_expr=NULL; - struct Maat_region_inner* u_para=NULL; - int expr_id=0, ret=0; - - u_para=Maat_region_inner_new(db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, -1); - ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_ip_rule->group_id, db_ip_rule->region_id, table->table_id, u_para); - if(ret!=0) - { - Maat_region_inner_free(u_para); - u_para=NULL; - return -1; - } - - expr_id=scanner->exprid_generator++; - Maat_region_inner_add_expr_id(u_para, expr_id); - op_expr=create_op_expr(expr_id, - RULESCAN_OP_ADD, - u_para, - table->table_id); - p_rule=create_rs_ip_rule(make_sub_type(table->table_id,CHARSET_NONE,0) - ,db_ip_rule); - op_expr_add_rule(op_expr, p_rule); - MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); - return 0; -} -int add_interval_rule(struct Maat_table_schema* table,struct db_interval_rule* intval_rule,struct Maat_scanner *scanner,void* logger) -{ - scan_rule_t* p_rule=NULL; - struct op_expr_t* op_expr=NULL; - struct Maat_region_inner* u_para=NULL; - int expr_id=0, ret=0,district_id=-1; - - if(table->table_type==TABLE_TYPE_INTERVAL_PLUS) - { - assert(strlen(intval_rule->district)>0); - str_unescape(intval_rule->district); - district_id=get_district_id(scanner, intval_rule->district); - } - - u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, district_id); - ret=Maat_hierarchy_add_region_to_group(scanner->hier, intval_rule->group_id, intval_rule->region_id, table->table_id, u_para); - if(ret!=0) - { - Maat_region_inner_free(u_para); - u_para=NULL; - return -1; - } - - expr_id=scanner->exprid_generator++; - Maat_region_inner_add_expr_id(u_para, expr_id); - - op_expr=create_op_expr(expr_id, - RULESCAN_OP_ADD, - u_para, - table->table_id); - - p_rule=create_rs_intval_rule(make_sub_type(table->table_id,CHARSET_NONE,0) - ,intval_rule); - op_expr_add_rule(op_expr,p_rule); - MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); - return 0; -} -int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger) -{ - struct Maat_region_inner* u_para=NULL; - struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - int expr_id=0, ret=0; - - u_para=Maat_region_inner_new(db_rule->group_id, db_rule->region_id, table->table_id, -1); - ret=Maat_hierarchy_add_region_to_group(scanner->hier, db_rule->group_id, db_rule->region_id, table->table_id, u_para); - if(ret!=0) - { - Maat_region_inner_free(u_para); - u_para=NULL; - return -1; - } - - expr_id=scanner->exprid_generator++; - Maat_region_inner_add_expr_id(u_para, expr_id); - - Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para); - return 0; -} - - -int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger) -{ - int i=0; - struct Maat_table_runtime* table_rt=NULL; - struct op_expr_t* op_expr=NULL; - - UNUSED int ret=0; - struct Maat_region_inner* region=NULL; - region=(struct Maat_region_inner*)Maat_hierarchy_region_dettach_user_data(maat_scanner->hier, region_id); - if(region==NULL) - { - return -1; - } - ret=Maat_hierarchy_remove_region_from_group(maat_scanner->hier, group_id, region_id); - assert(ret==0); - assert(group_id==region->group_id); - - switch(table->table_type) - { - case TABLE_TYPE_IP: - case TABLE_TYPE_IP_PLUS: - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_INTERVAL: - case TABLE_TYPE_INTERVAL_PLUS: - for(i=0;iexpr_id_cnt;i++) - { - op_expr=create_op_expr(region->expr_id_lb+i, RULESCAN_OP_DEL, NULL, table->table_id);//del expr - op_expr->rule_type=rule_type; - MESA_lqueue_join_tail(maat_scanner->region_update_q, &op_expr, sizeof(void*)); - } - break; - case TABLE_TYPE_SIMILARITY: - case TABLE_TYPE_DIGEST: - assert(region->expr_id_cnt==1); - table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); - Maat_table_runtime_digest_del(table_rt, region->expr_id_lb); - break; - default: - assert(0); - break; - } - Maat_garbage_bagging(maat_scanner->ref_garbage_bin, region, (void (*)(void*))Maat_region_inner_free); - - return 0; -} -void update_group2compile_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger) -{ - struct db_group2compile_rule db_g2c_rule; - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - int ret=0; - char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; - memset(&db_g2c_rule, 0, sizeof(db_g2c_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%s\t%d", &(db_g2c_rule.group_id), - &(db_g2c_rule.compile_id), - &(db_g2c_rule.is_valid), - &(db_g2c_rule.not_flag), - virtual_table_name, - &(db_g2c_rule.clause_index)); - if(ret!=6) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error, invalid format of group2compile table %s:%s", - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - return; - } - if(db_g2c_rule.clause_index>=MAX_ITEMS_PER_BOOL_EXPR) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error, invalid clause index of group2compile table %s:%s", - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - return; - } - if(is_valid_table_name(virtual_table_name)) - { - db_g2c_rule.virtual_table_id=Maat_table_manager_get_id_by_name(table_mgr, virtual_table_name); - if(db_g2c_rule.virtual_table_id<0) - { - //This happens when one data source (e.g. redis) is consumed by multiple Maat instance. - //Maat ignores unrealated groups. - MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module, - "group2compile table load abandon, unknown virtual table name: %s of group table %s:%s.", - virtual_table_name, - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - return; - } - } - if(db_g2c_rule.is_valid==FALSE) - { - ret=Maat_hierarchy_remove_group_from_compile(scanner->hier, db_g2c_rule.group_id,db_g2c_rule.virtual_table_id, db_g2c_rule.not_flag, db_g2c_rule.clause_index, db_g2c_rule.compile_id); - if(ret==0) - { - table_rt->origin_rule_num--; - assert(table_rt->origin_rule_num>=0); - if(db_g2c_rule.not_flag) - { - table_rt->group2compile.not_flag_group--; - } - } - } - else - { - ret=Maat_hierarchy_add_group_to_compile(scanner->hier, db_g2c_rule.group_id,db_g2c_rule.virtual_table_id, db_g2c_rule.not_flag, db_g2c_rule.clause_index, db_g2c_rule.compile_id); - if(ret==0) - { - table_rt->origin_rule_num++; - if(db_g2c_rule.not_flag) - { - table_rt->group2compile.not_flag_group++; - } - } - } - return; -} -void update_group2group_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger) -{ - struct db_group2group_rule db_g2g_rule; - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - int ret=0; - memset(&db_g2g_rule, 0, sizeof(db_g2g_rule)); - ret=sscanf(table_line,"%d\t%d\t%d", &(db_g2g_rule.group_id), - &(db_g2g_rule.superior_group_id), - &(db_g2g_rule.is_valid)); - if(ret!=3) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error, invalid format of group2group table %s:%s", - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - return; - } - if(db_g2g_rule.is_valid==FALSE) - { - ret=Maat_hierarchy_remove_group_from_group(scanner->hier, db_g2g_rule.group_id, db_g2g_rule.superior_group_id); - if(ret==0) - { - table_rt->origin_rule_num--; - assert(table_rt->origin_rule_num>=0); - } - } - else - { - ret=Maat_hierarchy_add_group_to_group(scanner->hier, db_g2g_rule.group_id, db_g2g_rule.superior_group_id); - if(ret==0) - { - table_rt->origin_rule_num++; - } - } - -} -void update_expr_rule(struct Maat_table_schema* table,const char* table_line,struct Maat_scanner *scanner,void* logger) -{ - struct db_expr_rule_t* maat_str_rule=ALLOC(struct db_expr_rule_t, 1); - int ret=0,db_hexbin=0,rule_type=0; - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - switch(table->table_type) - { - case TABLE_TYPE_EXPR: - ret=sscanf(table_line,"%d\t%d\t%1024s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) - ,&(maat_str_rule->group_id) - ,maat_str_rule->keywords - ,(int*)&(maat_str_rule->expr_type) - ,(int*)&(maat_str_rule->match_method) - ,&db_hexbin - ,&(maat_str_rule->is_valid)); - if(ret!=7) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "abandon config: invalid format of expr table %s:%s", table->table_name[table->updating_name], table_line); - free(maat_str_rule); - maat_str_rule=NULL; - table->udpate_err_cnt++; - return; - } - break; - case TABLE_TYPE_EXPR_PLUS: - ret=sscanf(table_line,"%d\t%d\t%64s\t%1024s\t%d\t%d\t%d\t%d",&(maat_str_rule->region_id) - ,&(maat_str_rule->group_id) - ,maat_str_rule->district - ,maat_str_rule->keywords - ,(int*)&(maat_str_rule->expr_type) - ,(int*)&(maat_str_rule->match_method) - ,&db_hexbin - ,&(maat_str_rule->is_valid)); - if(ret!=8) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "abandon config: invalid format of expr_plus table %s:%s", table->table_name[table->updating_name], table_line); - free(maat_str_rule); - maat_str_rule=NULL; - table->udpate_err_cnt++; - return; - } - break; - default: - assert(0); - break; - } - switch(db_hexbin) - { - case 0: - maat_str_rule->is_hexbin=FALSE; - maat_str_rule->is_case_sensitive=FALSE; - break; - case 1: - maat_str_rule->is_hexbin=TRUE; - maat_str_rule->is_case_sensitive=FALSE; - break; - case 2: - maat_str_rule->is_hexbin=FALSE; - maat_str_rule->is_case_sensitive=TRUE; - break; - default: - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "abandon config %d:update error,invalid hexbin value of expr table %s:%s", - maat_str_rule->region_id, - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - goto error_out; - } - if(!is_valid_match_method(maat_str_rule->match_method)) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "abandon config %d:update error,invalid match method=%d in expr table %s:%s", - maat_str_rule->region_id, - maat_str_rule->match_method, - table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - if(!is_valid_expr_type(maat_str_rule->expr_type)) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "abandon config %d:update error,invalid expr type=%d in expr table %s:%s", - maat_str_rule->region_id, - maat_str_rule->expr_type, - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - goto error_out; - } - - if(maat_str_rule->is_valid==FALSE) - { - if(maat_str_rule->expr_type==EXPR_TYPE_REGEX) - { - rule_type=RULETYPE_REG; - } - else - { - rule_type=RULETYPE_STR; - } - ret=del_region_rule(table, maat_str_rule->region_id, maat_str_rule->group_id, rule_type, - scanner, logger); - if(ret<0) - { - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num--; - } - } - else - { - if(maat_str_rule->expr_type==EXPR_TYPE_AND - &&maat_str_rule->match_method!=MATCH_METHOD_SUB) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.", - table->table_name[table->updating_name], maat_str_rule->region_id); - maat_str_rule->match_method=MATCH_METHOD_SUB; - - } - if(strlen(maat_str_rule->keywords)<4) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module , - "Table %s region cfg %d has a expr less than 4 bytes.", - table->table_name, - maat_str_rule->region_id); - } - ret=add_expr_rule(table, maat_str_rule, scanner, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, - "duplicate config of expr table %s region_id=%d", - table->table_name[table->updating_name], maat_str_rule->region_id); - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num++; - } - - } -error_out: - free(maat_str_rule); - maat_str_rule=NULL; -} - -void update_ip_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger) -{ - struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); - char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; - char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0}; - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0; - int protocol=0,direction=0; - int ret=0; - int ret_array[8]={1},i=0; - - switch(table->table_type) - { - case TABLE_TYPE_IP: - strncpy(saddr_format, "mask", sizeof(saddr_format)); - strncpy(sport_format, "mask", sizeof(sport_format)); - strncpy(daddr_format, "mask", sizeof(daddr_format)); - strncpy(dport_format, "mask", sizeof(dport_format)); - - ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d", - &(ip_rule->region_id), - &(ip_rule->group_id), - &(ip_rule->addr_type), - src_ip1, - src_ip2, - &src_port1, - &src_port2, - dst_ip1, - dst_ip2, - &dst_port1, - &dst_port2, - &protocol, - &direction, - &(ip_rule->is_valid)); - if(ret!=14) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, invalid column number of ip table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - break; - case TABLE_TYPE_IP_PLUS: - ret=sscanf(table_line,"%d\t%d\t%d\t%s\t%s\t%s\t%s\t%hu\t%hu\t%s\t%s\t%s\t%s\t%hu\t%hu\t%d\t%d\t%d", - &(ip_rule->region_id), - &(ip_rule->group_id), - &(ip_rule->addr_type), - saddr_format, - src_ip1, - src_ip2, - sport_format, - &src_port1, - &src_port2, - daddr_format, - dst_ip1, - dst_ip2, - dport_format, - &dst_port1, - &dst_port2, - &protocol, - &direction, - &(ip_rule->is_valid)); - if(ret!=18) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, invalid column number of ip_plus table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - break; - default: - table->udpate_err_cnt++; - goto error_out; - break; - } - if(ip_rule->addr_type!=4&&ip_rule->addr_type!=6) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "update error, invalid addr type %d of ip/ip_plus table %s:%s", - ip_rule->addr_type, - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - goto error_out; - } - if(protocol>65535 || protocol<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "update error, invalid protocol value %d of ip/ip_plus table %s:%s", - protocol, - table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - if(direction!=0 && direction!=1) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "update error, invalid direction value %d of ip/ip_plus table %s:%s", - direction, - table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - if(FORMAT_UNKNOWN==ip_format_str2int(saddr_format)|| - FORMAT_UNKNOWN==ip_format_str2int(sport_format)|| - FORMAT_UNKNOWN==ip_format_str2int(daddr_format)|| - FORMAT_UNKNOWN==ip_format_str2int(dport_format)) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, - "update error, invalid addr format of ip/ip_plus table %s:%s, should be range, mask or CIDR", - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; - goto error_out; - } - - if(ip_rule->addr_type==4) - { - ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, &ip_rule->ipv4_rule.min_saddr, &ip_rule->ipv4_rule.max_saddr); - ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, &ip_rule->ipv4_rule.min_daddr, &ip_rule->ipv4_rule.max_daddr); - - if(FORMAT_MASK==ip_format_str2int(sport_format)) - { - ip_rule->ipv4_rule.min_sport=src_port1&src_port2; - ip_rule->ipv4_rule.max_sport=src_port1|~src_port2; - } - else - { - ip_rule->ipv4_rule.min_sport=src_port1; - ip_rule->ipv4_rule.max_sport=src_port2; - } - - if(FORMAT_MASK==ip_format_str2int(dport_format)) - { - ip_rule->ipv4_rule.min_dport=dst_port1&dst_port2; - ip_rule->ipv4_rule.max_dport=dst_port1|~dst_port2; - } - else - { - ip_rule->ipv4_rule.min_dport=dst_port1; - ip_rule->ipv4_rule.max_dport=dst_port2; - } - - ip_rule->ipv4_rule.proto=protocol; - ip_rule->ipv4_rule.direction=direction; - } - else - { - ret_array[0]=ip_format2range(ip_rule->addr_type, ip_format_str2int(saddr_format), src_ip1, src_ip2, ip_rule->ipv6_rule.min_saddr, ip_rule->ipv6_rule.max_saddr); - ret_array[1]=ip_format2range(ip_rule->addr_type, ip_format_str2int(daddr_format), dst_ip1, dst_ip2, ip_rule->ipv6_rule.min_daddr, ip_rule->ipv6_rule.max_daddr); - - if(FORMAT_MASK==ip_format_str2int(sport_format)) - { - ip_rule->ipv6_rule.min_sport=src_port1&src_port2; - ip_rule->ipv6_rule.max_sport=src_port1|~src_port2; - } - else - { - ip_rule->ipv6_rule.min_sport=src_port1; - ip_rule->ipv6_rule.max_sport=src_port2; - } - - if(FORMAT_MASK==ip_format_str2int(dport_format)) - { - ip_rule->ipv6_rule.min_dport=dst_port1&dst_port2; - ip_rule->ipv6_rule.max_dport=dst_port1|~dst_port2; - } - else - { - ip_rule->ipv6_rule.min_dport=dst_port1; - ip_rule->ipv6_rule.max_dport=dst_port2; - } - - ip_rule->ipv6_rule.proto=protocol; - ip_rule->ipv6_rule.direction=direction; - } - for(i=0;i<4;i++) - { - if(ret_array[i]<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, invalid IP address format of ip table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - } - if(ip_rule->is_valid==FALSE) - { - ret=del_region_rule(table, - ip_rule->region_id, ip_rule->group_id, ip_rule->addr_type==6?RULETYPE_IPv6:RULETYPE_IPv4, - scanner, logger); - if(ret<0) - { - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num--; - } - } - else - { - - ret=add_ip_rule(table, ip_rule, scanner, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of ip table %s config_id=%d" - ,table->table_name[table->updating_name],ip_rule->region_id); - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num++; - } - - } -error_out: - free(ip_rule); - ip_rule=NULL; -} - -void update_intval_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger) -{ - struct db_interval_rule* intval_rule=ALLOC(struct db_interval_rule, 1); - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - int ret=0; - switch(table->table_type) - { - case TABLE_TYPE_INTERVAL: - ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) - ,&(intval_rule->group_id) - ,&(intval_rule->intval.lb) - ,&(intval_rule->intval.ub) - ,&(intval_rule->is_valid)); - - if(ret!=5||intval_rule->intval.ubintval.lb) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of interval table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - break; - case TABLE_TYPE_INTERVAL_PLUS: - ret=sscanf(table_line,"%d\t%d\t%s\t%u\t%u\t%d",&(intval_rule->region_id) - ,&(intval_rule->group_id) - ,intval_rule->district - ,&(intval_rule->intval.lb) - ,&(intval_rule->intval.ub) - ,&(intval_rule->is_valid)); - - if(ret!=6||intval_rule->intval.ubintval.lb) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of interval table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - break; - default: - assert(0); - break; - } - - - if(intval_rule->is_valid==FALSE) - { - ret=del_region_rule(table - ,intval_rule->region_id,intval_rule->group_id,RULETYPE_INT - ,scanner, logger); - if(ret<0) - { - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num--; - } - - } - else - { - ret=add_interval_rule(table, intval_rule,scanner,logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of intval table %s config_id=%d" - ,table->table_name[table->updating_name],intval_rule->region_id); - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num++; - } - - } -error_out: - free(intval_rule); - intval_rule=NULL; -} - -void update_compile_rule(struct Maat_table_schema* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger) -{ - struct compile_table_schema* compile_desc=&(table->compile); - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - - struct Maat_compile_rule *p_compile=NULL; - struct Maat_rule_head m_rule_tmp; - memset(&m_rule_tmp, 0, sizeof(m_rule_tmp)); - - char service_define[MAX_TABLE_LINE_SIZE]={0}; - char tag_str[MAX_TABLE_LINE_SIZE]={0}; - int ret=0; - int is_valid=0, declared_grp_num=0; - double exec_seq=0.0; - ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d\t%lf",&(m_rule_tmp.config_id), - &(m_rule_tmp.service_id), - &(m_rule_tmp.action), - &(m_rule_tmp.do_blacklist), - &(m_rule_tmp.do_log), - tag_str, - service_define, - &is_valid, - &declared_grp_num, - &exec_seq); - if((ret!=8&&ret!=9&&ret!=10)||declared_grp_num>MAAT_MAX_EXPR_ITEM_NUM) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module , - "update error, invalid format of compile table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - return; - - } - if(n_tags>0&&strlen(tag_str)>2) - { - str_unescape(tag_str); - ret=compare_accept_tag(tag_str, tags, n_tags); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error, invalid tag format of compile table %s:%s", - table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - return; - - } - if(ret==0) - { - table->unmatch_tag_cnt++; - return; - } - } - switch(compile_desc->user_region_encoding) - { - case USER_REGION_ENCODE_ESCAPE: - str_unescape(service_define); - break; - default: - break; - } - - if(is_valid==FALSE) - { - p_compile=(struct Maat_compile_rule*)Maat_hierarchy_compile_dettach_user_data(scanner->hier, m_rule_tmp.config_id); - if(p_compile) - { - ret=Maat_hierarchy_compile_remove(scanner->hier, m_rule_tmp.config_id); - assert(ret==0); - table_rt->origin_rule_num--; - Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); - } - else - { - table->udpate_err_cnt++; - } - } - else - { - p_compile=create_compile_rule(&m_rule_tmp, service_define, declared_grp_num, exec_seq, table); - ret=Maat_hierarchy_compile_add(scanner->hier, m_rule_tmp.config_id, declared_grp_num, p_compile); - if(ret==0) - { - table_rt->origin_rule_num++; - } - else - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, - "duplicate config of compile table %s compile_id %d", - table->table_name[table->updating_name], m_rule_tmp.config_id); - table->udpate_err_cnt++; - destroy_compile_rule(p_compile); - p_compile=NULL; - table->udpate_err_cnt++; - } - } - - return; -} - -void update_digest_rule(struct Maat_table_schema* table, const char* table_line, struct Maat_scanner *scanner, void* logger) -{ - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); - int ret=0; - char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; - if(table->table_type==TABLE_TYPE_DIGEST) - { - ret=sscanf(table_line,"%d\t%d\t%llu\t%s\t%hd\t%d",&(digest_rule->region_id) - ,&(digest_rule->group_id) - ,&(digest_rule->orgin_len) - ,digest_buff - ,&(digest_rule->confidence_degree) - ,&(digest_rule->is_valid)); - } - else if(table->table_type==TABLE_TYPE_SIMILARITY) - { - digest_rule->orgin_len=0; - ret=sscanf(table_line,"%d\t%d\t%s\t%hd\t%d",&(digest_rule->region_id) - ,&(digest_rule->group_id) - ,digest_buff - ,&(digest_rule->confidence_degree) - ,&(digest_rule->is_valid)); - } - else - { - assert(0); - } - digest_rule->digest_string=digest_buff; - if(!(ret==6||ret==5)||digest_rule->confidence_degree>100||digest_rule->confidence_degree<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid format of digest table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; - goto error_out; - } - - if(digest_rule->is_valid==FALSE) - { - //digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton. - ret=del_region_rule(table,digest_rule->region_id,digest_rule->group_id,0 ,scanner, logger); - if(ret<0) - { - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num--; - } - - } - else - { - ret=add_digest_rule(table, digest_rule,scanner,logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of intval table %s config_id=%d" - ,table->table_name[table->updating_name],digest_rule->region_id); - table->udpate_err_cnt++; - } - else - { - table_rt->origin_rule_num++; - } - - } -error_out: - digest_rule->digest_string=NULL; - free(digest_rule); - digest_rule=NULL; - return; -} -void update_generic_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) -{ - int ret=1, matched_tag=1; - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); - char* copy=NULL; - size_t accept_tag_offset=0, accept_tag_len=0; - int rule_tag_column=-1; - - switch(table_schema->table_type) - { - case TABLE_TYPE_PLUGIN: - rule_tag_column=table_schema->plugin.rule_tag_column; - break; - case TABLE_TYPE_IP_PLUGIN: - rule_tag_column=table_schema->ip_plugin.rule_tag_column; - break; - case TABLE_TYPE_FQDN_PLUGIN: - rule_tag_column=table_schema->fqdn_plugin.rule_tag_column; - break; - case TABLE_TYPE_BOOL_PLUGIN: - rule_tag_column=table_schema->bool_plugin.rule_tag_column; - break; - default: - assert(0); - break; - } - if(rule_tag_column>0&&n_tags>0) - { - ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of table %s:%s", - rule_tag_column, - table_schema->table_name[table_schema->updating_name], - table_row); - table_schema->udpate_err_cnt++; - return; - } - if(accept_tag_len>2) - { - copy=ALLOC(char, accept_tag_len+1); - memcpy(copy, table_row+accept_tag_offset, accept_tag_len); - matched_tag=compare_accept_tag(copy, tags, n_tags); - if(matched_tag<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error, invalid tag format of ip_plugin table_schema %s:%s", - table_schema->table_name[table_schema->updating_name], table_row); - table_schema->udpate_err_cnt++; - } - if(matched_tag==0) - { - table_schema->unmatch_tag_cnt++; - } - free(copy); - copy=NULL; - } - if(!matched_tag) - { - return; - } - } - switch(table_schema->table_type) - { - case TABLE_TYPE_PLUGIN: - Maat_table_runtime_plugin_new_row(table_rt, table_schema, table_row, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); - break; - case TABLE_TYPE_FQDN_PLUGIN: - Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); - break; - case TABLE_TYPE_BOOL_PLUGIN: - Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, table_row, logger); - break; - default: - assert(0); - break; - } - scanner->xx_plugin_rule_to_update_cnt++; - return; -} - -void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* logger) -{ - MESA_htable_handle tmp_map=NULL; - struct Maat_table_runtime* table_rt=NULL; - int i=0, ret=0; - enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; - - ret=Maat_hierarchy_rebuild(scanner->hier); - if(ret!=0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , - "Version %d hierarchy rebuild failed.", - scanner->version); - } - else - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "Version %d hierarchy rebuild success, dedup string rule %lu.", - scanner->version, - scanner->dedup_expr_num); - } - - scanner->dedup_expr_num=0; - rulescan_batch_update(scanner->region, - scanner->region_update_q, - logger, - scanner); - - for(i=0; (size_t)imax_table_num; i++) - { - table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); - if(table_rt==NULL) - { - continue; - } - table_type=Maat_table_runtime_get_type(table_rt); - switch(table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - ret=Maat_table_runtime_digest_batch_udpate(table_rt); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "GIE_update error."); - } - break; - case TABLE_TYPE_PLUGIN: - Maat_table_runtime_plugin_commit_update(table_rt); - break; - case TABLE_TYPE_IP_PLUGIN: - ret=Maat_table_runtime_ip_plugin_commit_update(table_rt); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "IP plugin table_id %d build failed.", i); - - } - - break; - case TABLE_TYPE_FQDN_PLUGIN: - ret=Maat_table_runtime_fqdn_plugin_commit_update(table_rt); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "FQDN plugin table_id %d build failed.", i); - - } - break; - case TABLE_TYPE_BOOL_PLUGIN: - ret=Maat_table_runtime_bool_plugin_commit_update(table_rt); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Boolean expression plugin table_id %d build failed.", i); - - } - break; - default: - break; - } - - } - if(scanner->tmp_district_map!=NULL) - { - tmp_map=scanner->district_map; - scanner->district_map=scanner->tmp_district_map; - scanner->tmp_district_map=NULL; - Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))maat_kv_store_free); - } - scanner->last_update_time=time(NULL); - scanner->xx_plugin_rule_to_update_cnt=0; - return; - -} - -void maat_start_cb(long long new_version, int update_type, void*u_para) -{ - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - feather->new_version=new_version; - - if(update_type==CM_UPDATE_TYPE_FULL) - { - feather->update_tmp_scanner=create_maat_scanner(new_version,feather); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Full config version %u -> %u update start", - feather->maat_version,new_version); - } - else - { - - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Inc config version %u -> %u update start", - feather->maat_version,new_version); - feather->maat_version=new_version; - } - Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type); - return; -} -long long scanner_rule_num(struct Maat_scanner *scanner) -{ - long long total=0; - struct Maat_table_runtime* table_rt=NULL; - int i=0; - for(i=0; (size_t)imax_table_num; i++) - { - table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); - if(table_rt!=NULL) - { - total+=table_rt->origin_rule_num; - } - } - return total; -} -void maat_finish_cb(void* u_para) -{ - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - long rulescan_wait_q_cnt=0; - - Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); - - if(feather->update_tmp_scanner!=NULL) - { - feather->update_tmp_scanner->cfg_num=scanner_rule_num(feather->update_tmp_scanner); - do_scanner_update(feather->update_tmp_scanner, - feather->scan_thread_num, - feather->logger); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Full config version %u load %d entries complete.", - feather->update_tmp_scanner->version,feather->update_tmp_scanner->cfg_num); - } - else if(feather->scanner!=NULL) - { - feather->scanner->cfg_num=scanner_rule_num(feather->scanner); - feather->scanner->version=feather->maat_version; - rulescan_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=rulescan_wait_q_cnt; - if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) - { - do_scanner_update(feather->scanner, - feather->scan_thread_num, - feather->logger); - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module - ,"Inc config version %u build complete, %d entries in total." - ,feather->scanner->version,feather->scanner->cfg_num); - feather->postpone_q_size=0; - } - else - { - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module, - "Postpone %d entries of version %u load to rulescan.", - feather->scanner->cfg_num, feather->scanner->version); - } - } - else - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Version %d have no valid scan rules, plugin callback complete.", - feather->maat_version); - } - feather->new_version=-1; - return; -} -int maat_update_cb(const char* table_name,const char* line,void *u_para) -{ - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - Maat_scanner* scanner=NULL; - struct Maat_table_schema* p_table=NULL; - if(feather->update_tmp_scanner!=NULL) - { - scanner=feather->update_tmp_scanner; - } - else - { - scanner=feather->scanner; - } -// MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); - p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, table_name); - if(!p_table) - { - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name); - return -1; - } - Maat_table_schema_set_updating_name(p_table, table_name); - - switch(p_table->table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - update_expr_rule(p_table, line, scanner, feather->logger); - break; - case TABLE_TYPE_IP: - case TABLE_TYPE_IP_PLUS: - update_ip_rule(p_table, line, scanner, feather->logger); - break; - case TABLE_TYPE_INTERVAL: - case TABLE_TYPE_INTERVAL_PLUS: - update_intval_rule(p_table, line, scanner,feather->logger); - break; - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - update_digest_rule(p_table, line, scanner,feather->logger); - break; - case TABLE_TYPE_COMPILE: - update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); - break; - case TABLE_TYPE_GROUP2COMPILE: - update_group2compile_rule(p_table, line, scanner, feather->table_mgr, feather->logger); - break; - case TABLE_TYPE_GROUP2GROUP: - update_group2group_rule(p_table, line, scanner, feather->table_mgr, feather->logger); - break; - case TABLE_TYPE_PLUGIN: - case TABLE_TYPE_IP_PLUGIN: - case TABLE_TYPE_FQDN_PLUGIN: - case TABLE_TYPE_BOOL_PLUGIN: - update_generic_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); - break; - default: - break; - - } - return 0; -} -void *thread_rule_monitor(void *arg) -{ - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; - struct Maat_scanner* old_scanner=NULL; - long rulescan_update_wait_q_cnt=0; - int scan_dir_cnt=0; - int ret=0; - char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; - char err_str[MAX_TABLE_NAME_LEN]={0}; - - struct stat attrib; - size_t total_wait_rule_cnt=0; - - char maat_name[16];//Defined by prctl: The name can be up to 16 bytes long,and should - // be null terminated if it contains fewer bytes. - if(strlen(feather->instance_name)>0) - { - snprintf(maat_name,sizeof(maat_name),"MAAT_%s",feather->instance_name); - } - else - { - snprintf(maat_name,sizeof(maat_name),"MAAT"); - } - ret=prctl(PR_SET_NAME,(unsigned long long)maat_name,NULL,NULL,NULL); - //pthread_setname_np are introduced in glibc2.12 - //ret=pthread_setname_np(pthread_self(),maat_name); - assert(ret>=0); - - pthread_mutex_lock(&(feather->background_update_mutex)); - if(feather->DEFERRED_LOAD_ON!=0) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Deferred Loading ON, updating in %s.",__func__); - maat_read_full_config(feather); - } - pthread_mutex_unlock(&(feather->background_update_mutex)); - while(feather->is_running) - { - usleep(feather->rule_update_checking_interval_ms*1000); - scan_dir_cnt++; - if(0==pthread_mutex_trylock(&(feather->background_update_mutex))) - { - switch(feather->input_mode) - { - case SOURCE_REDIS: - redis_monitor_traverse(feather->maat_version, - &(feather->mr_ctx), - maat_start_cb, - maat_update_cb, - maat_finish_cb, - feather, - feather->decrypt_key, //Not used. - feather); - break; - case SOURCE_IRIS_FILE: - config_monitor_traverse(feather->maat_version, - feather->iris_ctx.inc_dir, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - feather, - feather->decrypt_key, - feather->logger); - break; - case SOURCE_JSON_FILE: - memset(md5_tmp, 0, sizeof(md5_tmp)); - stat(feather->json_ctx.json_file, &attrib); - if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) - { - feather->json_ctx.last_md5_time=attrib.st_ctim; - md5_file(feather->json_ctx.json_file, md5_tmp); - if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5)) - { - ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str)); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s (md5=%s)failed: %s", - feather->json_ctx.json_file, - md5_tmp, - err_str); - } - else - { - config_monitor_traverse(0, - feather->json_ctx.iris_file, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - feather, - feather->decrypt_key, - feather->logger); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s success, md5: %s", - feather->json_ctx.json_file, - md5_tmp); - - } - - } - } - break; - default: - assert(0); - break; - } - - if(feather->update_tmp_scanner!=NULL) - { - old_scanner=feather->scanner; - //Some OS doesn't have __sync_lock_test_and_set. - //feather->scanner=__sync_lock_test_and_set(&(feather->scanner),feather->update_tmp_scanner); - feather->scanner=feather->update_tmp_scanner; - if(old_scanner!=NULL) - { - if(feather->scanner->version>old_scanner->version) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Maat version updated %d -> %d.", - old_scanner->version, feather->scanner->version); - } - else - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module, - "Maat version roll back %d -> %d.", - old_scanner->version, feather->scanner->version); - } - feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num); - Maat_garbage_bagging(feather->garbage_bin, old_scanner, (void (*)(void*))destroy_maat_scanner); - } - feather->update_tmp_scanner=NULL; - feather->maat_version=feather->scanner->version; - feather->last_full_version=feather->scanner->version; - } - if(feather->scanner!=NULL) - { - rulescan_update_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=rulescan_update_wait_q_cnt; - total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->xx_plugin_rule_to_update_cnt; - if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) - { - do_scanner_update(feather->scanner, - feather->scan_thread_num, - feather->logger); - feather->postpone_q_size=0; - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Actual udpate config version %u, %d entries load to rulescan after postpone.", - feather->scanner->version,feather->scanner->cfg_num); - } - } - pthread_mutex_unlock(&(feather->background_update_mutex)); - } - Maat_garbage_collect_routine(feather->garbage_bin); - if(feather->stat_on==1&&time(NULL)%2==0)//output every 2 seconds - { - maat_stat_output(feather); - } - } - - destroy_maat_scanner(feather->scanner); - Maat_garbage_bin_free(feather->garbage_bin); - Maat_table_manager_destroy(feather->table_mgr);//Table manager MUST be freed at last. - - alignment_int64_array_free(feather->thread_call_cnt); - alignment_int64_array_free(feather->compile_mid_cnt); - alignment_int64_array_free(feather->outer_mid_cnt); - alignment_int64_array_free(feather->hit_cnt); - alignment_int64_array_free(feather->not_grp_hit_cnt); - if(feather->input_mode==SOURCE_REDIS) - { - if(feather->mr_ctx.read_ctx) - { - redisFree(feather->mr_ctx.read_ctx); - feather->mr_ctx.read_ctx=NULL; - } - if(feather->mr_ctx.write_ctx) - { - redisFree(feather->mr_ctx.write_ctx); - feather->mr_ctx.write_ctx=NULL; - } - } - int i=0; - for(i=0; in_tags; i++) - { - free(feather->accept_tags[i].tag_name); - free(feather->accept_tags[i].tag_val); - } - free(feather->accept_tags); - free(feather->accept_tags_raw); - if(feather->stat_on&& feather->stat_handle) - { - FS_stop(&(feather->stat_handle)); - } - free(feather); - return NULL; -} - diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp deleted file mode 100644 index 78e0307..0000000 --- a/src/entry/Maat_stat.cpp +++ /dev/null @@ -1,368 +0,0 @@ -#include "Maat_rule_internal.h" -#include "Maat_table_schema.h" -#include "Maat_garbage_collection.h" -#include "alignment_int64.h" -#include -#include -enum MAAT_FS_STATUS{ - STATUS_VERSION=0, - STATUS_THRED_NUM, - STATUS_TABLE_NUM, - STATUS_PLUGIN_CACHE_NUM, - STATUS_PLUGIN_ACC_NUM, - STATUS_GROUP_REF_NUM, - STATUS_GROUP_REF_NOT_NUM, - STATUS_COMPILE_RULE_NUM, - STATUS_POSTPONE_QSIZE, - STATUS_OUTER_MID_NUM, - STATUS_INNER_MID_NUM, - STATUS_GARBAGE_QSIZE, - STATUS_TOTAL_SCAN_LEN, - STATUS_TOTAL_SCAN_CNT, - STATUS_UPDATE_ERR_CNT, - STATUS_ICONV_ERR_CNT, - STATUS_SCAN_ERR_CNT, - STATUS_ZOMBIE_RS_STREAM, - STATUS_NOT_GROUP_HIT, - STATUS_CMD_NUM, - STATUS_CMD_Q_SIZE, - STATUS_CMD_LINE_NUM -}; - -enum MAAT_FS_COLUMN -{ - COLUMN_TABLE_RULE_NUM=0, - COLUMN_TABLE_REGEX_NUM, - COLUMN_TABLE_STREAM_NUM, - COLUMN_TABLE_SCAN_CNT, - COLUMN_TABLE_SCAN_BYTES, - COLUMN_TABLE_CPU_TIME,//microseconds - COLUMN_TABLE_HIT_CNT, -}; -#define MAX_CONJ_NAME_LEN 22 -void maat_stat_init(struct _Maat_feather_t* feather) -{ - int value=0; - int i=0,j=0,offset=0; - struct Maat_table_schema* p_table=NULL; - char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0}; - - feather->stat_handle=FS_create_handle(); - FS_set_para(feather->stat_handle, OUTPUT_DEVICE, feather->stat_file, strlen(feather->stat_file)+1); - value=1; - FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value)); - value=0; - FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value)); - - FS_set_para(feather->stat_handle, APP_NAME, feather->instance_name, strlen(feather->instance_name)+1); - FS_set_para(feather->stat_handle, OUTPUT_PROMETHEUS, &feather->output_prometheus, sizeof(feather->output_prometheus)); - - feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "version"); - feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "threads"); - feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "tables"); - - feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "plug_cached"); - feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "plug_acc"); - - feather->fs_status_id[STATUS_GROUP_REF_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "group"); - feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "not_grp"); - feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "compile"); - - feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "postponed"); - feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "garbage_num"); - - feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "outer_mid"); - feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "inner_mid"); - feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "z_stream"); - - feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "nt_grp_hit"); - - feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_bytes"); - feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_times"); - feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "update_err"); - feather->fs_status_id[STATUS_ICONV_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "iconv_err"); - feather->fs_status_id[STATUS_SCAN_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "scan_error"); - feather->fs_status_id[STATUS_CMD_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "cmd_commit"); - feather->fs_status_id[STATUS_CMD_Q_SIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, "cmd_in_q"); - feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED, "line_cmd/s"); - - feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "rule"); - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "reg/v6"); - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT, "stream"); - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "IN_Bps"); - if(feather->perf_on==1) - { - feather->fs_column_id[COLUMN_TABLE_CPU_TIME]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "cpu_us"); - value=feather->fs_column_id[COLUMN_TABLE_CPU_TIME]; - FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value)); - FS_register_ratio(feather->stat_handle, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - 1000000, //microsecond to second - FS_STYLE_COLUMN, - FS_CALC_SPEED, - "PROC_Bps"); - } - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "IN_Tps"); - if(feather->perf_on==1) - { - FS_register_ratio(feather->stat_handle, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - 1000000, //microsecond to second - FS_STYLE_COLUMN, - FS_CALC_SPEED, - "PROC_Tps"); - } - feather->fs_column_id[COLUMN_TABLE_HIT_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED, "hit_cnt"); - value=feather->fs_column_id[COLUMN_TABLE_HIT_CNT]; - FS_set_para(feather->stat_handle, ID_INVISBLE, &value, sizeof(value)); - FS_register_ratio(feather->stat_handle, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - 1, - FS_STYLE_COLUMN, - FS_CALC_SPEED, - "hit_rate"); - feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum"); - size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); - for(i=0; i<(int)max_table_num; i++) - { - p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i); - if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN - ||p_table->table_type==TABLE_TYPE_GROUP - ||p_table->table_type==TABLE_TYPE_COMPILE) - { - continue; - } - offset=0; - for(j=0;jconj_cnt;j++) - { - offset+=snprintf(conj_table_name+offset,sizeof(conj_table_name)-offset, - "%s/", p_table->table_name[j]); - } - conj_table_name[offset-1]='\0';//delete the last slash - if(strlen(conj_table_name)>MAX_CONJ_NAME_LEN) - { - conj_table_name[MAX_CONJ_NAME_LEN]='\0'; - } - p_table->stat_line_id=FS_register(feather->stat_handle, - FS_STYLE_LINE, FS_CALC_CURRENT, - conj_table_name); - } - FS_start(feather->stat_handle); - return; -} - -void maat_stat_output(struct _Maat_feather_t* feather) -{ - long value=0; - long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; - long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; - long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; - long long outer_mid_cnt=0,inner_mid_cnt=0; - long long not_grp_hit_cnt=0; - long long total_update_error=0,total_iconv_error=0; - long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0; - int i=0; - time_t now; - struct Maat_table_schema* p_table=NULL; - struct Maat_table_runtime* table_rt=NULL; - time(&now); - if(feather->scanner==NULL) - { - return; - } - active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); - outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt, feather->scan_thread_num); - inner_mid_cnt=alignment_int64_array_sum(feather->compile_mid_cnt, feather->scan_thread_num); - not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt, feather->scan_thread_num); - - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0, FS_OP_SET, feather->maat_version); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0, FS_OP_SET, active_thread_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0, FS_OP_SET, Maat_table_manager_get_count(feather->table_mgr)); - - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0, FS_OP_SET, outer_mid_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0, FS_OP_SET, inner_mid_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0, FS_OP_SET, not_grp_hit_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0, FS_OP_SET, feather->cmd_acc_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0, FS_OP_SET, feather->cmd_q_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_LINE_NUM], 0, FS_OP_SET, feather->line_cmd_acc_num); - - value=Maat_garbage_bin_get_size(feather->garbage_bin); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); - feather->update_err_cnt=0; - feather->iconv_err_cnt=0; - size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); - for(i=0; i<(int)max_table_num; i++) - { - table_stream_num=0; - table_scan_cnt=0; - table_input_bytes=0; - table_scan_cpu_time=0; - table_hit_cnt=0; - table_regex_ipv6_num=0; - - p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i); - if(p_table==NULL) - { - continue; - } - table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i); - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - plugin_cache_num+=Maat_table_runtime_plugin_cached_row_count(table_rt); - plugin_acc_num+=table_rt->plugin.acc_line_num; - break; - case TABLE_TYPE_GROUP: - group_rule_num+=table_rt->origin_rule_num; - not_group_rule_num+=table_rt->group2compile.not_flag_group; - break; - case TABLE_TYPE_COMPILE: - compile_rule_num+=table_rt->origin_rule_num; - break; - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - table_regex_ipv6_num=table_rt->expr.regex_rule_cnt; - total_iconv_error=p_table->expr.iconv_err_cnt; - break; - case TABLE_TYPE_IP: - case TABLE_TYPE_IP_PLUS: - table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt; - break; - default: - break; - } - if(p_table->table_type==TABLE_TYPE_PLUGIN|| - p_table->table_type==TABLE_TYPE_GROUP|| - p_table->table_type==TABLE_TYPE_COMPILE) - { - continue; - } - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_RULE_NUM], - FS_OP_SET, - table_rt->origin_rule_num); - total_cfg_num+=table_rt->origin_rule_num; - - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], - FS_OP_SET, - table_regex_ipv6_num); - total_regex_num+=table_regex_ipv6_num; - - table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num); - alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], - FS_OP_ADD, - table_stream_num); - total_stream_cnt+= table_stream_num; - - - table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - FS_OP_ADD, - table_scan_cnt); - total_scan_cnt+=table_scan_cnt; - - table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num); - alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - FS_OP_ADD, - table_input_bytes); - total_input_bytes+=table_input_bytes; - if(feather->perf_on==1) - { - table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num); - alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num); - table_scan_cpu_time/=1000; - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - FS_OP_ADD, - table_scan_cpu_time); - total_cpu_time+=table_scan_cpu_time; - } - - table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - FS_OP_ADD, - table_hit_cnt); - total_update_error+=p_table->udpate_err_cnt; - } - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_RULE_NUM], - FS_OP_SET, - total_cfg_num); - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], - FS_OP_SET, - total_regex_num); - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], - FS_OP_ADD, - total_stream_cnt); - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - FS_OP_ADD, - total_scan_cnt); - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - FS_OP_ADD, - total_input_bytes); - if(feather->perf_on==1) - { - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - FS_OP_ADD, - total_cpu_time); - } - total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num); - alignment_int64_array_reset(feather->hit_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - feather->total_stat_id, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - FS_OP_ADD, - total_hit_cnt); - feather->total_scan_bytes+=total_input_bytes; - feather->total_scan_cnt+=total_scan_cnt; - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_LEN], 0,FS_OP_SET,feather->total_scan_bytes); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_CNT], 0,FS_OP_SET,feather->total_scan_cnt); - feather->update_err_cnt=total_update_error; - feather->iconv_err_cnt=total_iconv_error; - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_UPDATE_ERR_CNT], 0,FS_OP_SET,feather->update_err_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ICONV_ERR_CNT], 0,FS_OP_SET,feather->iconv_err_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream); - - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NUM], 0,FS_OP_SET,group_rule_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM], 0,FS_OP_SET,not_group_rule_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num); - - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size); - - FS_passive_output(feather->stat_handle); - return; -} - diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp deleted file mode 100644 index feb7bc2..0000000 --- a/src/entry/Maat_table_runtime.cpp +++ /dev/null @@ -1,980 +0,0 @@ -#include "Maat_rule.h" -#include "Maat_utils.h" -#include "Maat_table_runtime.h" -#include -#include -#include - - - -#include "dynamic_array.h" -#include -#include - -struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) -{ - struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); - - int ret[4]={0}; - size_t column_offset=0, column_len=0; - char start_ip[128]={0}, end_ip[128]={0}; - ret[0]=get_column_pos(row, schema->row_id_column, &column_offset, &column_len); - range_rule->rule_id=atoi(row+column_offset); - - ret[1]=get_column_pos(row, schema->ip_type_column, &column_offset, &column_len); - int ip_type=atoi(row+column_offset); - - ret[2]=get_column_pos(row, schema->start_ip_column, &column_offset, &column_len); - strncpy(start_ip, row+column_offset, MIN(column_len, sizeof(start_ip))); - - ret[3]=get_column_pos(row, schema->end_ip_column, &column_offset, &column_len); - strncpy(end_ip, row+column_offset, MIN(column_len, sizeof(end_ip))); - if(ret[0]<0||ret[1]<0||ret[2]<0||ret[3]<0) - { - free(range_rule); - return NULL; - } - - if(ip_type==4) - { - ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, &(range_rule->ipv4_rule.start_ip), &(range_rule->ipv4_rule.end_ip)); - range_rule->type=IPv4; - } - else if(ip_type==6) - { - ret[0]=ip_format2range(ip_type, FORMAT_RANGE, start_ip, end_ip, range_rule->ipv6_rule.start_ip, range_rule->ipv6_rule.end_ip); - range_rule->type=IPv6; - } - else - { - free(range_rule); - return NULL; - } - if(ret[0]<0) - { - free(range_rule); - return NULL; - } - range_rule->user_tag=NULL; - return range_rule; -} -void ip_rule_free(struct ip_rule* p) -{ - free(p); - return; -} -static int cmp_ull_p(const void *p1, const void *p2) -{ - if(* (unsigned long long*) p1 > * (unsigned long long*) p2) - { - return 1; - } - else if(* (unsigned long long*) p1 < * (unsigned long long*) p2) - { - return -1; - } - else - { - return 0; - } -} -size_t ull_dedup(unsigned long long item_ids[], size_t n_item) -{ - qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p); - - size_t J=0; - for(size_t i=1; iid=id; - rule->operation=op; - if(digest!=NULL) - { - digest_len=strlen(digest); - rule->sfh=(char*)calloc(sizeof(char),digest_len+1); - memcpy(rule->sfh,digest,digest_len); - - } - rule->sfh_length=digest_len; - rule->cfds_lvl=cfds_lvl; - rule->tag=tag; - return rule; -} -static void destroy_digest_rule(GIE_digest_t*rule) -{ - if(rule->sfh!=NULL) - { - free(rule->sfh); - rule->sfh=NULL; - } - free(rule); - rule=NULL; - return; -} -struct FQDN_rule *fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) -{ - struct FQDN_rule *fqdn_rule=ALLOC(struct FQDN_rule, 1); - //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$ - if(fqdn[0]=='.') - { - fqdn++; - fqdn_len--; - } - if(fqdn[fqdn_len]=='/') - { - fqdn_len--; - } - fqdn_rule->FQDN=ALLOC(char, fqdn_len+1); - memcpy(fqdn_rule->FQDN, fqdn, fqdn_len); - fqdn_rule->len=fqdn_len; - fqdn_rule->is_suffix_match=is_suffix_match; - fqdn_rule->id=id; - return fqdn_rule; -} -void fqdn_rule_free(struct FQDN_rule *fqdn_rule) -{ - assert(fqdn_rule->user_tag==NULL); - free(fqdn_rule->FQDN); - fqdn_rule->FQDN=NULL; - free(fqdn_rule); - return; -} -struct bool_expr *bool_expr_new(unsigned int id, unsigned long long item_ids[], size_t n_item) -{ - struct bool_expr *expr=ALLOC(struct bool_expr, 1); - expr->expr_id=id; - size_t i=0; - n_item=ull_dedup(item_ids, n_item); - for(i=0; iitems[i].item_id=item_ids[i]; - expr->items[i].not_flag=0; - } - expr->item_num=n_item; - return expr; -} -void bool_expr_free(struct bool_expr *expr) -{ - free(expr); - return; -} -void _notype_fqdn_rule_free(void* p) -{ - fqdn_rule_free((struct FQDN_rule*)p); - return; -} -static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num, struct Maat_garbage_bin* bin) -{ - - struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_schema->table_type; - switch(table_schema->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - table_rt->similar.update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->plugin.ex_schema.key2index_func, - NULL); - if(table_schema->plugin.ex_schema.set_flag) - { - EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); - } - break; - case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->ip_plugin.ex_schema.key2index_func, - free); - if(table_schema->ip_plugin.ex_schema.set_flag) - { - EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); - } - break; - case TABLE_TYPE_FQDN_PLUGIN: - table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->fqdn_plugin.ex_schema.key2index_func, - _notype_fqdn_rule_free); - if(table_schema->fqdn_plugin.ex_schema.set_flag) - { - EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema); - } - break; - case TABLE_TYPE_BOOL_PLUGIN: - table_rt->bool_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->fqdn_plugin.ex_schema.key2index_func, - (void (*)(void*))bool_expr_free); - if(table_schema->bool_plugin.ex_schema.set_flag) - { - EX_data_rt_set_schema(table_rt->bool_plugin.ex_data_rt, &table_schema->bool_plugin.ex_schema); - } - break; - default: - break; - } - table_rt->ref_garbage_bin=bin; - table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); - table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); - table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); - table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return table_rt; -} -static void table_runtime_free(struct Maat_table_runtime* p) -{ - long q_cnt=0,data_size=0; - int i=0; - UNUSED int q_ret=0; - - GIE_digest_t* digest_rule=NULL; - if(p==NULL) - { - return; - } - switch(p->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - if(p->similar.gie_handle!=NULL) - { - GIE_destory(p->similar.gie_handle); - } - if(p->similar.update_q!=NULL) - { - q_cnt=MESA_lqueue_get_count(p->similar.update_q); - for(i=0;isimilar.update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); - } - break; - - case TABLE_TYPE_IP_PLUGIN: - ip_matcher_free(p->ip_plugin.ip_matcher); - EX_data_rt_free(p->ip_plugin.ex_data_rt); - break; - case TABLE_TYPE_FQDN_PLUGIN: - FQDN_engine_free(p->fqdn_plugin.fqdn_engine); - EX_data_rt_free(p->fqdn_plugin.ex_data_rt); - break; - case TABLE_TYPE_PLUGIN: - EX_data_rt_free(p->plugin.ex_data_rt); - break; - default: - break; - } - - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - free(p); - return; -} - -struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin) -{ - const struct Maat_table_schema* table_desc=NULL; - struct Maat_table_runtime* table_rt=NULL; - struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1); - size_t i=0; - table_rt_mgr->ref_bin=bin; - table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager); - table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt); - for(i=0; in_table_rt; i++) - { - table_desc=Maat_table_manager_get_by_id_raw(table_manager, i); - if(!table_desc) - { - continue; - } - table_rt=table_runtime_new(table_desc, max_thread_num, table_rt_mgr->ref_bin); - table_rt_mgr->table_rt[i]=table_rt; - } - return table_rt_mgr; -} -void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr) -{ - size_t i=0; - for(i=0; in_table_rt; i++) - { - table_runtime_free(table_rt_mgr->table_rt[i]); - table_rt_mgr->table_rt[i]=NULL; - } - free(table_rt_mgr->table_rt); - table_rt_mgr->table_rt=NULL; - table_rt_mgr->ref_bin=NULL; - free(table_rt_mgr); -} -struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id) -{ - assert(table_id<(int)table_rt_mgr->n_table_rt); - return table_rt_mgr->table_rt[table_id]; -} -enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* table_rt) -{ - return table_rt->table_type; -} -void Maat_table_runtime_perf_stat(struct Maat_table_runtime* p, int scan_len, struct timespec* start, struct timespec* end,int thread_num) -{ - alignment_int64_array_add(p->scan_cnt,thread_num,1); - alignment_int64_array_add(p->input_bytes,thread_num,scan_len); - if(start!=NULL&&end!=NULL) - { - alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); - } - return; -} - -size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt) -{ - struct plugin_runtime* plugin_rt=&(table_rt->plugin); - assert(table_rt->table_type==TABLE_TYPE_PLUGIN); - return EX_data_rt_get_cached_row_num(plugin_rt->ex_data_rt); -} -const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row) -{ - const char* line=NULL; - struct plugin_runtime* plugin_rt=&(table_rt->plugin); - line=EX_data_rt_cached_row_get(plugin_rt->ex_data_rt, Nth_row); - return line; -} -MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key) -{ - MAAT_RULE_EX_DATA ex_data=NULL; - if(!table_schema->plugin.ex_schema.set_flag) - { - assert(0); - return NULL; - } - ex_data=EX_data_rt_get_EX_data_by_key(table_rt->plugin.ex_data_rt, key, strlen(key)); - return ex_data; - -} -void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) -{ - int ret=0, i=0; - size_t is_valid_offset=0, valid_len=0; - size_t key_offset=0, key_len=0; - - struct plugin_table_schema* plugin_schema=&table_schema->plugin; - struct plugin_runtime* plugin_rt=&table_rt->plugin; - ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); - plugin_rt->acc_line_num++; - if(plugin_schema->ex_schema.set_flag) - { - ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "plugin EX data process error: cannot find column %d of %s", - plugin_schema->key_column, row); - return; - } - if(atoi(row+is_valid_offset)==1) - { - EX_data_rt_row2EX_data(plugin_rt->ex_data_rt, row, row+key_offset, key_len, NULL, logger); - } - else - { - EX_data_rt_delete_by_row(plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger); - } - } - if(plugin_schema->cb_plug_cnt>0) - { - for(i=0; icb_plug_cnt;i++) - { - plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para); - } - } - if(!plugin_schema->ex_schema.set_flag && !plugin_schema->cb_plug_cnt) - { - EX_data_rt_cache_row_put(plugin_rt->ex_data_rt, row); - } - - return; -} -void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt) -{ - EX_data_rt_update_commit(table_rt->plugin.ex_data_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt); - return; -} - -void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) -{ - GIE_digest_t* digest_rule=NULL; - char *dup_digest=_maat_strdup(digest); - - if(table_rt->table_type==TABLE_TYPE_SIMILARITY) - { - dup_digest=str_unescape(dup_digest); - } - digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT, - dup_digest, - confidence_degree, - tag); - MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); - free(dup_digest); - return; -} -void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id) -{ - GIE_digest_t* digest_rule=NULL; - digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest - ,NULL - ,0 - ,NULL); - MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); - return; -} - -void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) -{ - struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin); - struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); - size_t is_valid_offset=0, valid_len=0; - size_t is_suffix_flag_offset=0, is_suffix_flag_len=0; - size_t row_id_offset=0, row_id_len=0; - size_t fqdn_offset=0, fqdn_len=0; - struct FQDN_rule* fqdn_rule=NULL; - int ret=0; - if(fqdn_plugin_schema->ex_schema.set_flag) - { - ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "fqdn_plugin EX data process error: cannot find is_valid column %d of %s", - fqdn_plugin_schema->valid_flag_column, row); - return; - } - ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "fqdn_plugin EX data process error: cannot find row id column %d of %s", - fqdn_plugin_schema->row_id_column, row); - return; - } - ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s", - fqdn_plugin_schema->is_suffix_flag_column, row); - return; - } - ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "fqdn_plugin EX data process error: cannot find fqdn column %d of %s", - fqdn_plugin_schema->fqdn_column, row); - return; - } - - if(atoi(row+is_valid_offset)==1)//add - { - fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset)); - ret=EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger); - if(ret<0) - { - fqdn_rule_free(fqdn_rule); - fqdn_rule=NULL; - } - } - else - { - EX_data_rt_delete_by_row(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); - } - } - else - { - EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row); - table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt); - } - fqdn_plugin_rt->changed_flag=1; - return; -} - -int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt) -{ - struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL; - struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; - assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); - struct EX_data_container **exc_array=NULL; - struct FQDN_rule* rules=NULL; - size_t rule_cnt=0, i=0, ret=0; - if(!fqdn_rt->changed_flag) - { - return ret; - } - rule_cnt=EX_data_rt_list_updating_ex_containers(fqdn_rt->ex_data_rt, &exc_array); - rules=ALLOC(struct FQDN_rule, rule_cnt); - for(i=0; i0) - { - new_fqdn_engine=FQDN_engine_new(rules, rule_cnt); - if(!new_fqdn_engine) - { - ret=-1; - } - } - old_fqdn_engine=fqdn_rt->fqdn_engine; - fqdn_rt->fqdn_engine=new_fqdn_engine; - Maat_garbage_bagging(table_rt->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free); - EX_data_rt_update_commit(fqdn_rt->ex_data_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(fqdn_rt->ex_data_rt); - - free(rules); - free(exc_array); - table_rt->fqdn_plugin.changed_flag=0; - return ret; -} -struct EX_data_rt *Maat_table_runtime_get_EX_data_rt(struct Maat_table_runtime* table_rt) -{ - struct EX_data_rt *rt=NULL; - - switch(table_rt->table_type) - { - case TABLE_TYPE_PLUGIN: - rt=table_rt->plugin.ex_data_rt; - break; - case TABLE_TYPE_IP_PLUGIN: - rt=table_rt->ip_plugin.ex_data_rt; - break; - case TABLE_TYPE_FQDN_PLUGIN: - rt=table_rt->fqdn_plugin.ex_data_rt; - break; - case TABLE_TYPE_BOOL_PLUGIN: - rt=table_rt->bool_plugin.ex_data_rt; - break; - default: - break; - } - return rt; -} -int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) -{ - size_t i=0; - const char* row=NULL; - assert(table_rt->table_type==table_schema->table_type); - struct EX_data_rt *ex_data_rt=NULL; - struct EX_data_schema *ex_data_schema=Maat_table_schema_get_EX_data_schema(table_schema); - ex_data_rt=Maat_table_runtime_get_EX_data_rt(table_rt); - EX_data_rt_set_schema(ex_data_rt, ex_data_schema); - for(i=0; itable_type) - { - case TABLE_TYPE_PLUGIN: - Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger); - break; - case TABLE_TYPE_FQDN_PLUGIN: - Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger); - break; - case TABLE_TYPE_BOOL_PLUGIN: - Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, row, logger); - break; - default: - break; - } - } - EX_data_rt_clear_row_cache(ex_data_rt); - switch(table_rt->table_type) - { - case TABLE_TYPE_PLUGIN: - Maat_table_runtime_plugin_commit_update(table_rt); - break; - case TABLE_TYPE_IP_PLUGIN: - Maat_table_runtime_ip_plugin_commit_update(table_rt); - break; - case TABLE_TYPE_FQDN_PLUGIN: - Maat_table_runtime_fqdn_plugin_commit_update(table_rt); - break; - case TABLE_TYPE_BOOL_PLUGIN: - Maat_table_runtime_bool_plugin_commit_update(table_rt); - break; - default: - break; - } - return 0; -} - -int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) -{ - struct FQDN_match results[size]; - int n_result=0, i=0; - if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) - { - return -1; - } - if(!table_rt->fqdn_plugin.fqdn_engine) - { - return 0; - } - n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size); - for(i=0; ifqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag); - } - return n_result; -} - -void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) -{ - struct bool_plugin_table_schema *bool_plugin_schema=&(table_schema->bool_plugin); - struct bool_plugin_runtime *bool_plugin_rt=&(table_rt->bool_plugin); - size_t is_valid_offset=0, valid_len=0; - size_t row_id_offset=0, row_id_len=0; - size_t bool_expr_offset=0, bool_expr_len=0; - struct bool_expr *expr=NULL; - int ret=0; - unsigned long long item_id[MAX_ITEMS_PER_BOOL_EXPR]; - size_t n_item=0; - if(bool_plugin_schema->ex_schema.set_flag) - { - ret=Maat_helper_read_column(row, bool_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "bool_plugin EX data process error: cannot find is_valid column %d of %s", - bool_plugin_schema->valid_flag_column, row); - return; - } - ret=Maat_helper_read_column(row, bool_plugin_schema->row_id_column, &row_id_offset, &row_id_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "bool_plugin EX data process error: cannot find row id column %d of %s", - bool_plugin_schema->row_id_column, row); - return; - } - ret=Maat_helper_read_column(row, bool_plugin_schema->bool_expr_column, &bool_expr_offset, &bool_expr_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "bool_plugin EX data process error: cannot find bool_expr column %d of %s", - bool_plugin_schema->bool_expr_column, row); - return; - } - char *token=NULL, *sub_token=NULL, *saveptr; - char expr_buffer[256]; - memset(expr_buffer, 0, sizeof(expr_buffer)); - memcpy(expr_buffer, row+bool_expr_offset, bool_expr_len); - for (token = expr_buffer; ; token= NULL) - { - sub_token= strtok_r(token, "&", &saveptr); - if (sub_token == NULL) - break; - ret=sscanf(sub_token, "%llu", item_id+n_item); - n_item++; - if(ret!=1||n_item>MAX_ITEMS_PER_BOOL_EXPR) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "bool_plugin EX data process error: invalid format of bool_expr column %d of %s", - bool_plugin_schema->bool_expr_column, row); - return; - } - } - if(atoi(row+is_valid_offset)==1)//add - { - expr=bool_expr_new((unsigned int)atoi(row+row_id_offset), item_id, n_item); - ret=EX_data_rt_row2EX_data(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, expr, logger); - if(ret<0) - { - bool_expr_free(expr); - expr=NULL; - } - } - else - { - EX_data_rt_delete_by_row(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); - } - } - else - { - EX_data_rt_cache_row_put(bool_plugin_rt->ex_data_rt, row); - table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(bool_plugin_rt->ex_data_rt); - } - bool_plugin_rt->changed_flag=1; - return; -} -int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt) -{ - struct bool_matcher* new_bool_matcher=NULL, *old_bool_matcher=NULL; - struct bool_plugin_runtime* bool_plugin_rt=&table_rt->bool_plugin; - assert(table_rt->table_type==TABLE_TYPE_BOOL_PLUGIN); - struct EX_data_container **exc_array=NULL; - struct bool_expr* exprs=NULL; - size_t expr_cnt=0, i=0, ret=0, mem_usage=0; - if(!bool_plugin_rt->changed_flag) - { - return ret; - } - expr_cnt=EX_data_rt_list_updating_ex_containers(bool_plugin_rt->ex_data_rt, &exc_array); - exprs=ALLOC(struct bool_expr, expr_cnt); - for(i=0; i0) - { - new_bool_matcher=bool_matcher_new(exprs, expr_cnt, &mem_usage); - if(!new_bool_matcher) - { - ret=-1; - } - } - old_bool_matcher=bool_plugin_rt->matcher; - bool_plugin_rt->matcher=new_bool_matcher; - Maat_garbage_bagging(table_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))bool_matcher_free); - EX_data_rt_update_commit(bool_plugin_rt->ex_data_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(bool_plugin_rt->ex_data_rt); - - free(exprs); - free(exc_array); - table_rt->fqdn_plugin.changed_flag=0; - return ret; -} - -int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) -{ - struct bool_expr_match results[size]; - int n_result=0; - if(table_rt->table_type!=TABLE_TYPE_BOOL_PLUGIN) - { - return -1; - } - if(!table_rt->bool_plugin.matcher) - { - return 0; - } - n_item=ull_dedup(item_ids, n_item); - - n_result=bool_matcher_match(table_rt->bool_plugin.matcher, item_ids, n_item, results, size); - for(int i=0; ibool_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag); - } - return n_result; -} -int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) -{ - long i=0,data_size=0; - int ret=0; - GIE_digest_t* digest_rule=NULL; - GIE_digest_t** update_array=NULL; - UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; - - - GIE_create_para_t para; - para.gram_value=7; - para.position_accuracy=10; - - const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); - if(q_cnt==0) - { - return 0; - } - if(table_rt->similar.gie_handle==NULL) - { - if(table_rt->table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - table_rt->similar.gie_handle=GIE_create(¶); - } - - update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); - for(i=0;isimilar.update_q, &digest_rule, &data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - update_array[i]=digest_rule; - digest_rule=NULL; - } - ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt); - for(i=0;ioperation==GIE_INSERT_OPT) - { - table_rt->origin_rule_num++; - } - else - { - table_rt->origin_rule_num--; - } - destroy_digest_rule(update_array[i]); - update_array[i]=NULL; - } - free(update_array); - update_array=NULL; - - if(ret!=(int)q_cnt) - { - return -1; - } - return q_cnt; -} - -int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt) -{ - struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL; - size_t rule_cnt=0; - size_t i=0; - struct ip_rule *rules=NULL; - struct EX_data_container **exc_array=NULL; - struct ip_plugin_runtime *ip_plugin=&(table_rt->ip_plugin); - int ret=0; - assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); - if(!ip_plugin->changed_flag) - { - return ret; - } - rule_cnt=EX_data_rt_list_updating_ex_containers(ip_plugin->ex_data_rt, &exc_array); - rules=ALLOC(struct ip_rule, rule_cnt); - for(i=0; i0) - { - new_ip_matcher=ip_matcher_new(rules, rule_cnt, &ip_plugin->mem_use_by_ip_matcher); - if(!new_ip_matcher) - { - ret=-1; - } - } - old_ip_matcher=ip_plugin->ip_matcher; - ip_plugin->ip_matcher=new_ip_matcher; - Maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); - EX_data_rt_update_commit(ip_plugin->ex_data_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(ip_plugin->ex_data_rt); - - free(rules); - free(exc_array); - exc_array=NULL; - ip_plugin->changed_flag=0; - return ret; -} - - -void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) -{ - struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); - struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); - size_t is_valid_offset=0, valid_len=0; - size_t row_id_offset=0, row_id_len=0; - struct ip_rule* ip_rule=NULL; - int ret=0; - if(ip_plugin_schema->ex_schema.set_flag) - { - ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "ip_plugin EX data process error: cannot find is_valid column %d of %s", - ip_plugin_schema->row_id_column, row); - return; - } - ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "ip_plugin EX data process error: cannot find row id column %d of %s", - ip_plugin_schema->row_id_column, row); - return; - } - ip_rule=ip_plugin_row2ip_rule(ip_plugin_schema, row); - if(ip_rule==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "ip_plugin read ip error: %s", row); - return; - - } - if(atoi(row+is_valid_offset)==1)//add - { - ret=EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger); - if(ret<0) - { - ip_rule_free(ip_rule); - ip_rule=NULL; - } - } - else - { - - ret=EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); - ip_rule_free(ip_rule); - ip_rule=NULL; - } - } - else - { - EX_data_rt_cache_row_put(ip_plugin_rt->ex_data_rt, row); - table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(ip_plugin_rt->ex_data_rt); - } - ip_plugin_rt->changed_flag=1; - return; -} -int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) -{ - struct scan_result results[size]; - int n_result=0, i=0; - if(!table_rt->ip_plugin.ip_matcher) - { - return 0; - } - n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size); - for(i=0; iip_plugin.ex_data_rt, (struct EX_data_container *)results[i].tag); - } - return n_result; -} - - diff --git a/src/entry/Maat_table_schema.cpp b/src/entry/Maat_table_schema.cpp deleted file mode 100644 index 60191a1..0000000 --- a/src/entry/Maat_table_schema.cpp +++ /dev/null @@ -1,1256 +0,0 @@ -#include "Maat_table_schema.h" -#include "map_str2int.h" -#include "Maat_utils.h" -#include "cJSON.h" - -#include -#include -#include -#include -#include - -#define MAX_TABLE_NUM 256 - -struct Maat_table_manager -{ - struct Maat_table_schema* p_table_info[MAX_TABLE_NUM]; - size_t table_cnt; - struct maat_kv_store* map_tablename2id; - int active_plugin_table_num; - int is_last_plugin_table_updating; - void* logger; -}; -enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type) -{ - enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID; - switch(table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_SIMILARITY: - case TABLE_TYPE_DIGEST: - ret=SCAN_TYPE_STRING; - break; - case TABLE_TYPE_INTERVAL: - case TABLE_TYPE_INTERVAL_PLUS: - ret=SCAN_TYPE_INTERVAL; - break; - case TABLE_TYPE_IP: - case TABLE_TYPE_IP_PLUS: - case TABLE_TYPE_COMPOSITION: - ret=SCAN_TYPE_IP; - break; - case TABLE_TYPE_PLUGIN: - ret=SCAN_TYPE_PLUGIN; - break; - case TABLE_TYPE_IP_PLUGIN: - ret=SCAN_TYPE_IP; - break; - case TABLE_TYPE_FQDN_PLUGIN: - ret=SCAN_TYPE_FQDN_PLUGIN; - break; - case TABLE_TYPE_BOOL_PLUGIN: - ret=SCAN_TYPE_BOOL_PLUGIN; - break; - case TABLE_TYPE_COMPILE: - ret=SCAN_TYPE_NONE; - break; - default: - break; - } - return ret; -} - -int read_expr_table_info(const char* line, struct Maat_table_schema* table, struct maat_kv_store* string2int_map) -{ - int j=0, ret[4]={0}; - char table_type[16], src_charset[256], dst_charset[256], merge[4], quick_str_scan[32]={0}; - char *token=NULL, *sub_token=NULL, *saveptr; - struct string_table_schema* p=&(table->expr); - sscanf(line, "%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id), - table->table_name[0], - table_type, - src_charset, - dst_charset, - merge, - &(p->cross_cache_size), - quick_str_scan); - memset(ret, 0, sizeof(ret)); - - ret[0]=maat_kv_read(string2int_map, table_type, (int*)&(table->table_type)); - ret[1]=maat_kv_read(string2int_map, src_charset, (int*)&(p->src_charset)); - ret[2]=maat_kv_read(string2int_map, merge, &(p->do_charset_merge)); - if(strlen(quick_str_scan)>0) - { - ret[3]=maat_kv_read(string2int_map, quick_str_scan, &(p->quick_expr_switch)); - } - memset(quick_str_scan, 0, sizeof(quick_str_scan)); - - for(j=0; j<4; j++) - { - if(ret[j]<0) - { - return -1; - } - } - j=0; - for (token = dst_charset; ; token= NULL) - { - sub_token= strtok_r(token, "/", &saveptr); - if (sub_token == NULL) - break; - ret[3]=maat_kv_read(string2int_map, sub_token, (int*)&(p->dst_charset[j])); - if(ret[3]>0) - { - if(p->dst_charset[j]==p->src_charset) - { - p->src_charset_in_dst=TRUE; - } - j++; - } - else - { - return -1; - } - - } - return 0; -} - -Maat_table_schema* table_info_new(void) -{ - struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1); - p->conj_cnt=1; - return p; -} -void table_info_free(struct Maat_table_schema*p) -{ - free(p); - return; -} -int _read_integer_arrary(char* string, int *array, int size) -{ - char *token=NULL,*sub_token=NULL,*saveptr; - int i=0; - for (token = string, i=0; itype!=cJSON_Array) - { - goto error_out; - } - cJSON_ArrayForEach(tmp, json) - { - if(tmp->type!=cJSON_String) - { - goto error_out; - } - ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id); - if(ret<0) - { - goto error_out; - } - physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; - physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); - if(physical_table_scan_typevirtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; - } - } - else //For compatible non-json physical description - { - ret=maat_kv_read(table_mgr->map_tablename2id, json_str, &tmp_table_id); - if(ret<0) - { - goto error_out; - } - physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; - physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); - table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; - } - cJSON_Delete(json); - free(copy_line); - return 0; - -error_out: - if(json) cJSON_Delete(json); - free(copy_line); - return -1; - -} - -int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) -{ - int i=0,ret=0; - size_t offset=0, len=0; - cJSON* json=NULL, *tmp=NULL, *array_item=NULL; - char* copy_line=NULL, *plug_info=NULL; - struct plugin_table_schema* plugin_desc=&(p->plugin); - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_PLUGIN_SCHEMA_JSON, &offset, &len); - if(ret<0) - { - goto error_out; - } - if(offset+lenvalid_flag_column)); - if(ret==0||ret==EOF) - { - plugin_desc->valid_flag_column=-1; - } - free(copy_line); - return 0; - } - json=cJSON_Parse(plug_info); - if(!json) - { - goto error_out; - } - tmp=cJSON_GetObjectItem(json, "key"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->key_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->valid_flag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->rule_tag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "foreign"); - if(tmp!=NULL) - { - if(tmp->type==cJSON_String) - { - plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); - } - else if(tmp->type==cJSON_Array) - { - plugin_desc->n_foreign= cJSON_GetArraySize(tmp); - for(i=0;in_foreign; i++) - { - array_item=cJSON_GetArrayItem(tmp, i); - assert(array_item->type==cJSON_Number); - plugin_desc->foreign_columns[i]=array_item->valueint; - } - } - } - cJSON_Delete(json); - - free(copy_line); - return 0; -error_out: - free(copy_line); - return -1; -} -int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p) -{ - int ret=0, read_cnt=0; - size_t offset=0, len=0; - cJSON* json=NULL, *tmp=NULL; - char* copy_line=NULL, *ip_plugin_info=NULL; - struct ip_plugin_table_schema* ip_plugin_schema=&(p->ip_plugin); - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_IP_PLUGIN_SCHEMA_JSON, &offset, &len); - if(ret<0) - { - goto error_out; - } - if(offset+lentype==cJSON_Number) - { - ip_plugin_schema->row_id_column=tmp->valueint; - read_cnt++; - } - - tmp=cJSON_GetObjectItem(json, "ip_type"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - ip_plugin_schema->ip_type_column=tmp->valueint; - read_cnt++; - } - tmp=cJSON_GetObjectItem(json, "start_ip"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - ip_plugin_schema->start_ip_column=tmp->valueint; - read_cnt++; - } - tmp=cJSON_GetObjectItem(json, "end_ip"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - ip_plugin_schema->end_ip_column=tmp->valueint; - read_cnt++; - } - - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - ip_plugin_schema->valid_flag_column=tmp->valueint; - read_cnt++; - } - ip_plugin_schema->rule_tag_column=-1; - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - ip_plugin_schema->rule_tag_column=tmp->valueint; - //read_cnt++; Tag is optional, so NOT ++ intentionally. - } - - cJSON_Delete(json); - - free(copy_line); - if(read_cnt<5) - { - return -1; - } - else - { - return 0; - } -error_out: - free(copy_line); - return -1; - -} -int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p) -{ - int ret=0, read_cnt=0; - size_t offset=0, len=0; - cJSON* json=NULL, *tmp=NULL; - char* copy_line=NULL, *fqnd_plugin_schema_json=NULL; - struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin); - - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SCHEMA_JSON, &offset, &len); - if(ret<0) - { - goto error_out; - } - if(offset+lentype==cJSON_Number) - { - fqdn_plugin_schema->row_id_column=tmp->valueint; - read_cnt++; - } - - tmp=cJSON_GetObjectItem(json, "is_suffix_match"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint; - read_cnt++; - } - tmp=cJSON_GetObjectItem(json, "fqdn"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - fqdn_plugin_schema->fqdn_column=tmp->valueint; - read_cnt++; - } - - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - fqdn_plugin_schema->valid_flag_column=tmp->valueint; - read_cnt++; - } - fqdn_plugin_schema->rule_tag_column=-1; - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - fqdn_plugin_schema->rule_tag_column=tmp->valueint; - //read_cnt++; Tag is optional, so NOT ++ intentionally. - } - - cJSON_Delete(json); - - free(copy_line); - if(read_cnt<4) - { - return -1; - } - else - { - return 0; - } -error_out: - free(copy_line); - return -1; - -} -int read_bool_plugin_table_schema(const char* line, struct Maat_table_schema* p) -{ - int ret=0, read_cnt=0; - size_t offset=0, len=0; - cJSON* json=NULL, *tmp=NULL; - char* copy_line=NULL, *schema_json=NULL; - struct bool_plugin_table_schema* bool_plugin_schema=&(p->bool_plugin); - - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_BOOL_PLUGIN_SCHEMA_JSON, &offset, &len); - if(ret<0) - { - goto error_out; - } - if(offset+lentype==cJSON_Number) - { - bool_plugin_schema->row_id_column=tmp->valueint; - read_cnt++; - } - tmp=cJSON_GetObjectItem(json, "bool_expr"); - if(tmp!=NULL && tmp->type==cJSON_Number) - { - bool_plugin_schema->bool_expr_column=tmp->valueint; - read_cnt++; - } - - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - bool_plugin_schema->valid_flag_column=tmp->valueint; - read_cnt++; - } - bool_plugin_schema->rule_tag_column=-1; - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - bool_plugin_schema->rule_tag_column=tmp->valueint; - //read_cnt++; Tag is optional, so NOT ++ intentionally. - } - - cJSON_Delete(json); - - free(copy_line); - if(read_cnt<3) - { - return -1; - } - else - { - return 0; - } -error_out: - free(copy_line); - return -1; - -} - -int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map) -{ - int ret=0; - size_t offset=0, len=0; - cJSON* json=NULL, *tmp=NULL; - char* copy_line=NULL, *composition_info=NULL; - struct composition_table_schema* composition_schema=&(p->composition); - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_COMPOSITION_SCHEMA_JSON, &offset, &len); - if(ret<0) - { - goto error_out; - } - if(offset+lentype==cJSON_String) - { - ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP])); - if(ret<0) - { - MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - tmp->valuestring, - p->table_name[0], - p->table_id); - goto error_out; - } - } - tmp=cJSON_GetObjectItem(json, "destination"); - if(tmp!=NULL && tmp->type==cJSON_String) - { - ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP])); - if(ret<0) - { - MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - tmp->valuestring, - p->table_name[0], - p->table_id); - goto error_out; - } - - } - tmp=cJSON_GetObjectItem(json, "session"); - if(tmp!=NULL && tmp->type==cJSON_String) - { - ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION])); - if(ret<0) - { - MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - tmp->valuestring, - p->table_name[0], - p->table_id); - goto error_out; - } - } - cJSON_Delete(json); - free(copy_line); - return 0; - -error_out: - free(copy_line); - return -1; - -} -void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) -{ - size_t i=0; - for(i=0;ip_table_info[i]==NULL) - { - continue; - } - table_info_free(table_mgr->p_table_info[i]); - table_mgr->p_table_info[i]=NULL; - } - maat_kv_store_free(table_mgr->map_tablename2id); - free(table_mgr); - return; -} - -struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger) -{ - struct Maat_table_manager* table_mgr=NULL; - FILE*fp=NULL; - char line[MAX_TABLE_LINE_SIZE]; - int i=0, ret=0; - char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - struct maat_kv_store* reserved_word_map=NULL;; - struct Maat_table_schema*p=NULL; - struct Maat_table_schema*conj_table=NULL; - fp=fopen(table_info_path,"r"); - if(fp==NULL) - { - fprintf(stderr,"Maat read table info %s error.\n",table_info_path); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); - return NULL; - } - table_mgr=ALLOC(struct Maat_table_manager, 1); - struct Maat_table_schema** p_table_info=table_mgr->p_table_info; - size_t n_table=MAX_TABLE_NUM; - table_mgr->logger=logger; - table_mgr->map_tablename2id=maat_kv_store_new(); - - reserved_word_map=maat_kv_store_new(); - maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); - maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP); - maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); - maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); - maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); - maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); - maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); - maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); - maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); - maat_kv_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); - maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); - maat_kv_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); - maat_kv_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST); - maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); - maat_kv_register(reserved_word_map, "group", TABLE_TYPE_GROUP); - maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); - maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); - maat_kv_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY); - maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); - maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); - maat_kv_register(reserved_word_map, "quickoff", 0); - maat_kv_register(reserved_word_map, "quickon", 1); - maat_kv_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE); -// maat_kv_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet - - const char** charset_name_list=charset_get_all_name(); - for(i=0;i0) - { - maat_kv_register(reserved_word_map, charset_name_list[i], i); - } - else - { - break; - } - } - - maat_kv_register(reserved_word_map,"yes", 1); - maat_kv_register(reserved_word_map,"no", 0); - - - i=0; - while(NULL!=fgets(line,sizeof(line),fp)) - { - i++; - - if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4) - { - continue; - } - p=table_info_new(); - - ret=sscanf(line, "%d\t%s\t%s\t%[a-z0-9\t ]", &(p->table_id), - p->table_name[0], - table_type_str, - not_care); - if(ret<3) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: not enough column.",table_info_path,i); - continue; - } - ret=maat_kv_read(reserved_word_map, table_type_str, (int*)&(p->table_type)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error:invalid table type.",table_info_path,i); - goto invalid_table; - } - switch(p->table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, p, reserved_word_map); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal column.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal column.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_schema(line, p); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal plugin table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: illegal plugin table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_IP_PLUGIN: - ret=read_ip_plugin_table_schema(line, p); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal ip_plugin table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: illegal ip_plugin table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_FQDN_PLUGIN: - ret=read_fqdn_plugin_table_schema(line, p); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal fqdn_plugin table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: illegal fqdn_plugin table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_BOOL_PLUGIN: - ret=read_bool_plugin_table_schema(line, p); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal bool_plugin table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: illegal bool_plugin table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_COMPOSITION: - ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal composition table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: illegal composition table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_VIRTUAL: - ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map); - if(ret<0) - { - fprintf(stderr, "Maat read table info %s line %d error: illegal virtual table schema.\n", table_info_path, i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: illegal virtual table schema.", table_info_path, i); - goto invalid_table; - } - break; - case TABLE_TYPE_COMPILE: - ret=sscanf(not_care,"%[a-z0-9]", tmp_str); - if(ret>0) - { - ret=maat_kv_read(reserved_word_map, tmp_str, (int*)&(p->compile.user_region_encoding)); - } - if(ret!=1) - { - p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; - } - break; - default: - break; - } - - if((unsigned int)p->table_id>=n_table) - { - fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table); - - goto invalid_table; - } - ret=maat_kv_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Duplicate table %s of table id %d", - p->table_name[0], - p->table_id); - goto invalid_table; - } - if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; - { - conj_table=p_table_info[p->table_id]; - if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error: reach tableid %d conjunction upper limit." - ,table_info_path,i,p->table_id); - goto invalid_table; - } - memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); - conj_table->conj_cnt++; - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, - "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." - ,table_info_path,i,p->table_name[0] - ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); - //use goto to free the conjunctioned table_info - goto invalid_table; - } - p_table_info[p->table_id]=p; - table_mgr->table_cnt++; - continue; - -invalid_table: - table_info_free(p); - p=NULL; - } - fclose(fp); - maat_kv_store_free(reserved_word_map); - return table_mgr; -} -size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr) -{ - return MAX_TABLE_NUM; -} -size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr) -{ - return table_mgr->table_cnt; -} -int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) -{ - int i=0; - for(i=0; i< MAX_TABLE_NUM; i++) - { - if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) - { - strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); - return 1; - } - } - return 0; -} -int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) -{ - int i=0; - for(i=0; i< MAX_TABLE_NUM; i++) - { - if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2COMPILE) - { - strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); - return 1; - } - } - return 0; -} -int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) -{ - int i=0; - for(i=0; i< MAX_TABLE_NUM; i++) - { - if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP2GROUP) - { - strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); - return 1; - } - } - return 0; -} -enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) -{ - if(table_id>MAX_TABLE_NUM) - { - return TABLE_TYPE_INVALID; - } - if(table_mgr->p_table_info[table_id]) - { - return table_mgr->p_table_info[table_id]->table_type; - } - return TABLE_TYPE_INVALID; - -} - -struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id) -{ - enum MAAT_SCAN_TYPE tab_scan_type; - struct Maat_table_schema **p_table_info=table_mgr->p_table_info; - size_t n_table=MAX_TABLE_NUM; - - struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL; - if((unsigned int) table_id>n_table) - { - return NULL; - } - if(p_table_info[table_id]==NULL) - { - return NULL; - } - p_table=p_table_info[table_id]; - if(p_table==NULL) - { - return NULL; - } - if(p_table->table_type==TABLE_TYPE_VIRTUAL) - { - p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]]; - *virutal_table_id=table_id; - } - else - { - p_physical_table=p_table; - if(virutal_table_id) *virutal_table_id=0; - } - tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type); - if(tab_scan_type!=scan_type) - { - return NULL; - } - return p_physical_table; -} -int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) -{ - int table_id=-1,ret=0; - ret=maat_kv_read(table_mgr->map_tablename2id, table_name, &table_id); - if(ret>0) - { - return table_id; - } - else - { - return -1; - } -} -int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr, - int table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para) -{ - int idx=0; - struct Maat_table_schema *p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); - struct plugin_table_schema *plugin_desc=&(p_table->plugin); - if(p_table==NULL) - { - return -1; - } - - idx=plugin_desc->cb_plug_cnt; - if(idx==MAX_PLUGIN_PER_TABLE) - { - return -1; - } - plugin_desc->cb_plug_cnt++; - plugin_desc->cb_plug[idx].start=start; - plugin_desc->cb_plug[idx].update=update; - plugin_desc->cb_plug[idx].finish=finish; - plugin_desc->cb_plug[idx].u_para=u_para; - return 1; -} - -struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) -{ - int table_id=-1; - struct Maat_table_schema *p_table=NULL; - - table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name); - if(table_id<0) - { - return NULL; - } - p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); - if(!p_table) - { - return NULL; - } - if(idxcompile.ex_data_num) - { - return p_table->compile.ex_desc+idx; - } - return NULL; - -} -int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, - Maat_rule_EX_new_func_t *new_func, - Maat_rule_EX_free_func_t* free_func, - Maat_rule_EX_dup_func_t* dup_func, - long argl, void *argp) -{ - int table_id=-1; - struct Maat_table_schema *p_table=NULL; - table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name); - if(table_id<0) - { - return -1; - } - p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); - if(!p_table) - { - return -1; - } - int idx=-1; - - struct compile_table_schema* compile_desc=&(p_table->compile); - if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) - { - return -1; - } - idx=compile_desc->ex_data_num; - compile_desc->ex_desc[idx].idx=idx; - compile_desc->ex_desc[idx].table_id=table_id; - compile_desc->ex_desc[idx].argl=argl; - compile_desc->ex_desc[idx].argp=argp; - compile_desc->ex_desc[idx].new_func=new_func; - compile_desc->ex_desc[idx].free_func=free_func; - compile_desc->ex_desc[idx].dup_func=dup_func; - - compile_desc->ex_data_num++; - - return idx; -} -struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema) -{ - struct EX_data_schema *ex_schema=NULL; - switch(table_schema->table_type) - { - case TABLE_TYPE_PLUGIN: - ex_schema=&table_schema->plugin.ex_schema; - break; - case TABLE_TYPE_IP_PLUGIN: - ex_schema=&table_schema->ip_plugin.ex_schema; - break; - case TABLE_TYPE_FQDN_PLUGIN: - ex_schema=&table_schema->fqdn_plugin.ex_schema; - break; - case TABLE_TYPE_BOOL_PLUGIN: - ex_schema=&table_schema->bool_plugin.ex_schema; - break; - default: - break; - } - return ex_schema; -} - -int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger) -{ - struct EX_data_schema *ex_schema=NULL; - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } - ex_schema=Maat_table_schema_get_EX_data_schema(table_schema); - if(ex_schema==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a valid plugin table.", __FUNCTION__); - return -1; - } - if(ex_schema->set_flag) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data schema already registed.", __FUNCTION__); - return -1; - } - ex_schema->new_func=new_func; - ex_schema->free_func=free_func; - ex_schema->dup_func=dup_func; - ex_schema->key2index_func=key2index_func;//Set but not used. - ex_schema->argl=argl; - ex_schema->argp=argp; - ex_schema->set_flag=1; - return 0; - -} -void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) -{ - table_mgr->active_plugin_table_num=0; - int i=0, j=0; - struct Maat_table_schema* p_table=NULL; - struct plugin_table_schema* plugin_desc=NULL; - - for(i=0; ip_table_info[i]; - plugin_desc=&(p_table->plugin); - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) - { - continue; - } - - table_mgr->active_plugin_table_num++; - - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].start!=NULL) - { - plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para); - } - } - } - -} -void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr) -{ - int i=0, j=0; - struct Maat_table_schema* p_table=NULL; - struct plugin_table_schema* plugin_desc=NULL; - - int call_plugin_table_cnt=0; - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - plugin_desc=&(p_table->plugin); - call_plugin_table_cnt++; - if(call_plugin_table_cnt==table_mgr->active_plugin_table_num) - { - table_mgr->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].finish!=NULL) - { - plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); - } - } - table_mgr->is_last_plugin_table_updating=0; - break; - default: - break; - } - } - table_mgr->active_plugin_table_num=0; - return; -} -int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr) -{ - return table_mgr->is_last_plugin_table_updating; -} -struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) -{ - struct Maat_table_schema * p_table=NULL; - int table_id=0; - table_id=Maat_table_manager_get_id_by_name(table_mgr, table_name); - if(table_id<0) - { - return NULL; - } - p_table=table_mgr->p_table_info[table_id]; - return p_table; -} -void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name) -{ - int i=0; - for(i=0; iconj_cnt; i++) - { - if(0==strcmp(p_table->table_name[i], table_name)) - { - p_table->updating_name=i; - } - } - assert(i<=p_table->conj_cnt); -} -struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) -{ - if(table_id>MAX_TABLE_NUM||table_id<0) - { - return NULL; - } - - return table_mgr->p_table_info[table_id]; -} - -int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type) -{ - int ret=-1; - struct Maat_table_schema* p_table=Maat_table_manager_get_by_id_raw(table_mgr, parent_table_id); - if(p_table->table_type!=TABLE_TYPE_COMPOSITION) - { - return -1; - } - ret=p_table->composition.component_table_id[type]; - return ret; -} -int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table) -{ - int valid_flag_column=-1; - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - valid_flag_column=p_table->plugin.valid_flag_column; - break; - case TABLE_TYPE_IP_PLUGIN: - valid_flag_column=p_table->ip_plugin.valid_flag_column; - break; - case TABLE_TYPE_FQDN_PLUGIN: - valid_flag_column=p_table->fqdn_plugin.valid_flag_column; - break; - case TABLE_TYPE_BOOL_PLUGIN: - valid_flag_column=p_table->bool_plugin.valid_flag_column; - break; - default: - valid_flag_column=-1; - break; - } - return valid_flag_column; -} - diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp deleted file mode 100644 index 7195ce4..0000000 --- a/src/entry/Maat_utils.cpp +++ /dev/null @@ -1,594 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "Maat_utils.h" -pid_t gettid() -{ - return syscall(SYS_gettid); -} -const char* module_name_str(const char*name) -{ - static __thread char module[64]; - snprintf(module,sizeof(module),"%s(%d)", name, gettid()); - return module; -} - -int converHextoint(char srctmp) -{ - if(isdigit(srctmp)) - { - return srctmp-'0'; - } - else - { - char temp=toupper(srctmp); - temp=temp-'A'+10; - return temp; - } -} -int hex2bin(char *hex,int hex_len,char *binary,int size) -{ - int i=0; - int resultlen=0; - int high,low; - for(i=0;iresultlen; i+=2,resultlen++) - { - high=converHextoint(hex[i]); - low=converHextoint(hex[i+1]); - binary[resultlen]=high*16+low; - } - size=resultlen; - binary[resultlen]='\0'; - return resultlen; -} -//functioned as strdup, for dictator compatible. -char* _maat_strdup(const char* s) -{ - char*d=NULL; - if(s==NULL) - { - return NULL; - } - d=(char*)malloc(strlen(s)+1); - memcpy(d,s,strlen(s)+1); - return d; -} -char* str_tolower(char* string) -{ - int i=0; - for(i=0;i<(int)strlen(string);i++) - { - string[i]=(char)tolower(string[i]); - } - return string; -} -char * strchr_esc(char* s,const char delim) -{ - char *token; - if(s==NULL) - return NULL; - for(token=s;*token!='\0';token++) - { - if(*token=='\\') - { - token++; - continue; - } - if(*token==delim) - break; - } - if (*token == '\0') - { - return NULL; - } - else - { - return token; - } -} -char *strtok_r_esc(char *s, const char delim, char **save_ptr) -{ - char *token; - - if (s == NULL) s = *save_ptr; - - /* Scan leading delimiters. */ - token=strchr_esc(s,delim); - if(token==NULL) - { - *save_ptr=token; - return s; - } - /* Find the end of the token. */ - *token='\0'; - token++; - *save_ptr=token; - - return s; -} -char *str_unescape_and(char*s) -{ - int i=0,j=0; - for(i=0,j=0;i<(int)strlen(s);i++) - { - if(s[i]=='\\'&&s[i+1]=='&') - { - s[j]='&'; - i++; - j++; - } - else{ - s[j]=s[i]; - j++; - } - } - s[j]='\0'; - return s; -} -char* str_unescape(char* s) -{ - int i=0,j=0; - int len=strlen(s); - for(i=0,j=0;i %s", src_file, dst_file); - return system(cmd); -} - -int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password) -{ - char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s", password, src_file, dst_file); - return system(cmd); -} -int system_cmd_rm(const char* src_file) -{ - char cmd[MAX_SYSTEM_CMD_LEN] = { 0 }; - snprintf(cmd,sizeof(cmd), "rm %s -f", src_file); - return system(cmd); -} -char* md5_file(const char* filename, char* md5string) -{ - FILE* fp=NULL; - int i=0; - unsigned char md5[MD5_DIGEST_LENGTH]; - struct stat file_info; - stat(filename, &file_info); - size_t file_size=file_info.st_size; - - fp=fopen(filename,"r"); - if(fp==NULL) - { - return NULL; - } - char* file_buff=(char*)malloc(file_size); - fread(file_buff,1,file_size,fp); - fclose(fp); - - MD5((const unsigned char *)(file_buff), (unsigned long)(file_size), md5); - for(i = 0; i < MD5_DIGEST_LENGTH; ++i) - { - sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]); - } - free(file_buff); - return md5string; -} -const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", - "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8", "windows-1251", ""}; - -const char** charset_get_all_name(void) -{ - return CHARSET_STRING; -} -const char* charset_get_name(enum MAAT_CHARSET charset) -{ - return CHARSET_STRING[charset]; -} -int lqueue_destroy_cb(void *data, long data_len, void *arg) -{ - assert(0); - return 0; -} - -int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz) -{ - int ret=0, out_blk_len=0; - int out_buff_len=0, out_buff_offset=0; - EVP_CIPHER_CTX *ctx; - - unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; - unsigned char cipher_iv[EVP_MAX_IV_LENGTH]; - memset(cipher_key,0,sizeof(cipher_key)); - memset(cipher_iv,0,sizeof(cipher_iv)); - - const EVP_CIPHER *cipher; - const EVP_MD *dgst=NULL; - const unsigned char *salt=NULL; - - OpenSSL_add_all_algorithms(); - cipher=EVP_get_cipherbyname(algorithm); - if(cipher==NULL) - { - snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm); - return 0; - } - dgst=EVP_get_digestbyname("md5"); - if(dgst==NULL) - { - snprintf(err_str, err_str_sz, "Get MD5 object failed."); - return 0; - } - ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key, cipher_iv); - if(ret==0) - { - snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); - return 0; - } - /* Don't set key or IV right away; we want to check lengths */ - ctx = EVP_CIPHER_CTX_new(); - EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt); - OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0); - OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); - - /* Now we can set key and IV */ - //It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call). - EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1); - out_buff_len=inlen+EVP_CIPHER_block_size(cipher)-1; - *pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char)); - if (!EVP_CipherUpdate(ctx, *pp_out+out_buff_offset, &out_blk_len, inbuf, inlen)) - { - snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed."); - EVP_CIPHER_CTX_free(ctx); - goto error_out; - } - out_buff_offset+=out_blk_len; - if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len)) - { - snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?"); - EVP_CIPHER_CTX_free(ctx); - goto error_out; - } - out_buff_offset+=out_blk_len; - EVP_CIPHER_CTX_free(ctx); - *out_sz=out_buff_offset; - return 0; - -error_out: - free(*pp_out); - *pp_out=NULL; - return -1; -} -int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz) -{ - int ret=0; - FILE* fp=NULL; - struct stat fstat_buf; - size_t read_size=0; - - ret=stat(file_name, &fstat_buf); - if(ret!=0) - { - return -1; - } - fp=fopen(file_name, "r"); - if(fp==NULL) - { - return -1; - } - *out_sz=fstat_buf.st_size; - *pp_out=(unsigned char*)calloc(1, *out_sz+1); - read_size=fread(*pp_out, 1, *out_sz, fp); - if(read_size!= *out_sz) - { - free(*pp_out); - pp_out=NULL; - return -1; - } - - fclose(fp); - fp=NULL; - return 0; -} -int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz) -{ - int ret=0; - size_t file_sz=0; - unsigned char* file_buff=NULL; - ret=load_file_to_memory(file_name, &file_buff, &file_sz); - if(ret<0) - { - return -1; - } - ret=crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz); - free(file_buff); - file_buff=NULL; - return ret; -} - -int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, - size_t *out_uncompressed_sz) -{ - z_stream strm; - strm.zalloc = NULL; - strm.zfree = NULL; - strm.opaque = NULL; - - strm.avail_in = in_compressed_sz; - strm.avail_out = *out_uncompressed_sz; - strm.next_in = (Bytef*) in_compressed_data; - strm.next_out = *out_uncompressed_data; - - int ret = -1; - ret = inflateInit2(&strm, MAX_WBITS+16); - if (ret == Z_OK) - { - ret = inflate(&strm, Z_FINISH); - if (ret == Z_STREAM_END) - { - *out_uncompressed_sz = strm.total_out; - ret = inflateEnd(&strm); - return ret; - } - } - inflateEnd(&strm); - return ret; -} -int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, - size_t *out_uncompressed_sz) -{ - - - int z_result; - int ret=-1; - size_t buffer_sz=in_compressed_sz*2; - *out_uncompressed_data = (unsigned char*) malloc(buffer_sz); - do{ - *out_uncompressed_sz=buffer_sz; - z_result = gzip_uncompress_one_try( - in_compressed_data, - in_compressed_sz, - out_uncompressed_data, - out_uncompressed_sz); - - switch( z_result ) - { - case Z_OK: - ret=0; - break; - case Z_BUF_ERROR: - buffer_sz*=2; - *out_uncompressed_data=(unsigned char*) realloc(*out_uncompressed_data, buffer_sz); - break; - default: - ret=-1; - break; - } - - }while(z_result==Z_BUF_ERROR); - return ret; -} - - -enum MAAT_IP_FORMAT ip_format_str2int(const char* format) -{ - if(0==strcasecmp(format, "range")) - { - return FORMAT_RANGE; - } - else if(0==strcasecmp(format, "mask")) - { - return FORMAT_MASK; - } - else if(0==strcasecmp(format, "CIDR")) - { - return FORMAT_CIDR; - } - else - { - assert(0); - } - return FORMAT_UNKNOWN; -} -int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]) -{ - unsigned int ipv4_addr=0, ipv4_mask=0, ipv4_range_end=0; - unsigned int ipv6_addr[4]={0}, ipv6_mask[4]={0}, ipv6_range_end[4]={0}; - int cidr=0, bit32=0; - int ret=0, i=0; - if(ip_type!=4 && ip_type!=6) - { - assert(0); - return -1; - } - if(ip_type==4) - { - ret=inet_pton(AF_INET, ip1, &ipv4_addr); - if(ret<=0) - { - return -1; - } - ipv4_addr=ntohl(ipv4_addr); - switch (format) - { - case FORMAT_RANGE: - range_begin[0]=ipv4_addr; - ret=inet_pton(AF_INET, ip2, &ipv4_range_end); - if(ret<=0) - { - return -1; - } - ipv4_range_end=ntohl(ipv4_range_end); - range_end[0]=ipv4_range_end; - break; - case FORMAT_MASK: - ret=inet_pton(AF_INET, ip2, &ipv4_mask); - if(ret<=0) - { - return -1; - } - ipv4_mask=ntohl(ipv4_mask); - range_begin[0]=ipv4_addr&ipv4_mask; - range_end[0]=ipv4_addr|~ipv4_mask; - break; - case FORMAT_CIDR: - cidr=atoi(ip2); - if(cidr>32||cidr<0) - { - return -1; - } - ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL; - range_begin[0]=ipv4_addr&ipv4_mask; - range_end[0]=ipv4_addr|~ipv4_mask; - break; - default: - assert(0); - } - } - else //ipv6 - { - ret=inet_pton(AF_INET6, ip1, ipv6_addr); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_addr); - switch(format) - { - case FORMAT_RANGE: - ret=inet_pton(AF_INET6, ip2, ipv6_range_end); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_range_end); - memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); - memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); - break; - case FORMAT_MASK: - ret=inet_pton(AF_INET6, ip2, ipv6_mask); - if(ret<=0) - { - return -1; - } - ipv6_ntoh(ipv6_mask); - for(i=0; i<4; i++) - { - range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; - range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; - } - break; - case FORMAT_CIDR: - cidr=atoi(ip2); - if(cidr>128||cidr<0) - { - return -1; - } - for(i=0; i<4; i++) - { - bit32=128-cidr-32*(3-i); - if(bit32<0) bit32=0; - ipv6_mask[i]=(0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL; - range_begin[i]=ipv6_addr[i]&ipv6_mask[i]; - range_end[i] = ipv6_addr[i]|~ipv6_mask[i]; - } - break; - default: - assert(0); - } - } - return 0; -} - diff --git a/src/entry/UniversalBoolMatch.cpp b/src/entry/UniversalBoolMatch.cpp deleted file mode 100644 index ff91d60..0000000 --- a/src/entry/UniversalBoolMatch.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include "UniversalBoolMatch.h" -#include -#include -#include -using namespace std; -#include -#include - -static const unsigned int MAX_ARRAY_SIZE=65536; - -struct thread_local_data_t -{ - unsigned int mapped_ids[MAX_ARRAY_SIZE]; - unsigned int used_cells[MAX_ARRAY_SIZE]; - unsigned char * bitmap; - unsigned int * matched_bitmap; -}; - -struct boolexpr_matcher_t -{ - unsigned int max_thread_num; - unsigned int bool_expr_num; - unsigned int multi_expr_num; - void ** bool_expr_ids; - unsigned char * multi_expr_size; - unsigned int bool_item_id_num; - unsigned int min_item_id; - unsigned int max_item_id; - unsigned int * bool_item_ids; - unsigned int * mapped_ptr; - unsigned int * mapped_ids; - unsigned int theta; - unsigned int L[65537]; - thread_local_data_t * thread_data; -}; - -void * boolexpr_initialize(universal_bool_expr_t * bool_exprs, unsigned int bool_expr_num, unsigned int max_thread_num, unsigned int * mem_size) -{ - if(bool_exprs==NULL || bool_expr_num==0 || max_thread_num==0) return NULL; - - for(unsigned int i=0; iMAX_ITEMS_PER_BOOL_EXPR) - { - return NULL; - } - } - - int I=-1, J=(int)bool_expr_num; - while(I1) I++; - if(I==J) break; - J--; - while(J>I && bool_exprs[J].bool_item_num==1) J--; - if(J==I) break; - swap(bool_exprs[I], bool_exprs[J]); - } - - for(int k=0; k<(int)bool_expr_num; k++) - { - if((k=I && bool_exprs[k].bool_item_num>1)) - { - printf("[%s:%d]: fatal error!\n", __FILE__, __LINE__); - return NULL; - } - } - - unsigned int mem_bytes=0; - - boolexpr_matcher_t * matcher=new boolexpr_matcher_t; - mem_bytes+=sizeof(boolexpr_matcher_t); - - matcher->max_thread_num=max_thread_num; - matcher->bool_expr_num=bool_expr_num; - matcher->multi_expr_num=I; - - matcher->bool_expr_ids=new void *[bool_expr_num]; - mem_bytes+=bool_expr_num*sizeof(void *); - - matcher->multi_expr_size=new unsigned char[matcher->multi_expr_num+1]; - mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char); - - matcher->thread_data=new thread_local_data_t[max_thread_num]; - mem_bytes+=max_thread_num*sizeof(thread_local_data_t); - - for(unsigned int i=0; ithread_data[i].bitmap=new unsigned char[matcher->multi_expr_num+1]; - mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char); - - unsigned int size=(bool_expr_num-matcher->multi_expr_num); - size=(size>>5)+1; - matcher->thread_data[i].matched_bitmap=new unsigned int[size]; - mem_bytes+=size*sizeof(unsigned int); - } - - map< unsigned int, vector > M; - unsigned int count=0; - for(unsigned int i=0; ibool_expr_ids[i] =bool_exprs[i].bool_expr_id; - if(imulti_expr_num) - { - matcher->multi_expr_size[i]=bool_exprs[i].bool_item_num; - } - count+=bool_exprs[i].bool_item_num; - for(unsigned int j=0; jbool_item_id_num=(unsigned int)M.size(); - matcher->bool_item_ids=new unsigned int[M.size()]; - matcher->mapped_ptr =new unsigned int[M.size()+1]; - matcher->mapped_ids =new unsigned int[count]; - mem_bytes+=(2*(unsigned int)M.size()+1+count)*sizeof(unsigned int); - - matcher->mapped_ptr[0]=0; - map< unsigned int, vector >::const_iterator it=M.begin(); - for(unsigned int k=0; kbool_item_ids[k]=it->first; - copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]); - matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size(); - } - - matcher->min_item_id=matcher->bool_item_ids[0]; - matcher->max_item_id=matcher->bool_item_ids[M.size()-1]; - for(unsigned int k=0; kbool_item_ids[k]-=matcher->min_item_id; - } - - unsigned long long ONE=1; - unsigned int theta=0; - while((ONE<<(theta+16))<=matcher->bool_item_ids[M.size()-1]) theta++; - matcher->theta=theta; - - matcher->L[0]=0; - for(unsigned int i=1; i<65536; i++) - { - matcher->L[i]=(unsigned int)(lower_bound(matcher->bool_item_ids, matcher->bool_item_ids+M.size(), i*(1U<bool_item_ids); - } - matcher->L[65536]=(unsigned int)M.size(); - - M.clear(); - - *mem_size=mem_bytes; - return matcher; -} - -int boolexpr_match(void * instance, unsigned int thread_id, unsigned int * item_ids, unsigned int item_num, void ** result, unsigned int size) -{ - if(instance==NULL) return -1; - - boolexpr_matcher_t * matcher=(boolexpr_matcher_t *)instance; - if(thread_id>=matcher->max_thread_num) return -1; - - unsigned int * mapped_ids=matcher->thread_data[thread_id].mapped_ids; - unsigned int ids_num=0; - for(unsigned int i=0; imin_item_id || item_ids[i]>matcher->max_item_id) continue; - - unsigned int id=item_ids[i]-matcher->min_item_id; - unsigned int k=id>>matcher->theta; - - int l=matcher->L[k], h=(int)matcher->L[k+1]-1; - if(hbool_item_ids[m]) h=m-1; - else l=m+1; - } - if(h<(int)matcher->L[k] || matcher->bool_item_ids[h]!=id) continue; - - for(unsigned int j=matcher->mapped_ptr[h]; jmapped_ptr[h+1]; j++) - { - if(ids_num==MAX_ARRAY_SIZE) return -1; - mapped_ids[ids_num++]=matcher->mapped_ids[j]; - } - } - - unsigned int * used_cells=matcher->thread_data[thread_id].used_cells; - unsigned int used_num=0; - for(unsigned int i=0; i>3); - } - - unsigned char * bitmap=matcher->thread_data[thread_id].bitmap; - unsigned int * matched_bitmap=matcher->thread_data[thread_id].matched_bitmap; - for(unsigned int i=0; imulti_expr_num) - { - bitmap[used_cells[i]]=0; - } - else - { - unsigned int j=used_cells[i]-matcher->multi_expr_num; - matched_bitmap[j>>5]&=~(1U<<(j&31)); - } - } - - unsigned int r=0; - - for(unsigned int i=0; i>3); - if(xmulti_expr_num) - { - unsigned int y=(mapped_ids[i]&7); - if((bitmap[x]&(1U<multi_expr_size[x])-1) - { - if(rbool_expr_ids[x]; - } - } - } - else - { - unsigned int j=x-matcher->multi_expr_num; - if((matched_bitmap[j>>5]&(1U<<(j&31)))==0) - { - if(rbool_expr_ids[x]; - matched_bitmap[j>>5]|=(1U<<(j&31)); - } - } - } - - return r; -} - -void boolexpr_destroy(void * instance) -{ - if(instance!=NULL) - { - boolexpr_matcher_t * matcher=(boolexpr_matcher_t *)instance; - delete [] matcher->bool_expr_ids; - delete [] matcher->multi_expr_size; - delete [] matcher->bool_item_ids; - delete [] matcher->mapped_ptr; - delete [] matcher->mapped_ids; - for(unsigned int i=0; imax_thread_num; i++) - { - delete [] matcher->thread_data[i].bitmap; - delete [] matcher->thread_data[i].matched_bitmap; - } - delete [] matcher->thread_data; - delete matcher; - } -} diff --git a/src/entry/bool_matcher.cpp b/src/entry/bool_matcher.cpp deleted file mode 100644 index c3992c3..0000000 --- a/src/entry/bool_matcher.cpp +++ /dev/null @@ -1,216 +0,0 @@ -#include "bool_matcher.h" -#include -#include -#include -using namespace std; -#include -#include -#include - -struct bool_expr_item -{ - size_t item_num; - struct bool_item * items; -}; - -struct bool_matcher -{ - unsigned int bool_expr_num; - struct bool_expr_match * bool_expr_ids; - struct bool_expr_item * bool_expr_items; - unsigned int bool_item_num; - unsigned long long * bool_items; - unsigned int * mapped_ptr; - unsigned int * mapped_ids; - unsigned int bitmap_size; - unsigned char * bitmap; -}; - -bool operator<(const struct bool_item & lhs, const struct bool_item & rhs) -{ - return lhs.item_idbool_expr_num=(unsigned int)expr_num; - matcher->bool_expr_ids =new struct bool_expr_match[expr_num]; - matcher->bool_expr_items=new struct bool_expr_item[expr_num]; - mem_bytes+=(unsigned int)expr_num*(sizeof(struct bool_expr_match)+sizeof(struct bool_expr_item)); - for(unsigned int i=0; ibool_expr_ids[i].expr_id =exprs[i].expr_id; - matcher->bool_expr_ids[i].user_tag =exprs[i].user_tag; - matcher->bool_expr_items[i].item_num=exprs[i].item_num; - matcher->bool_expr_items[i].items=new struct bool_item[exprs[i].item_num]; - mem_bytes+=(unsigned int)exprs[i].item_num*sizeof(struct bool_item); - copy(exprs[i].items, exprs[i].items+exprs[i].item_num, matcher->bool_expr_items[i].items); - sort(matcher->bool_expr_items[i].items, matcher->bool_expr_items[i].items+exprs[i].item_num); - } - - map M1; - for(unsigned int i=0; i > M2; - for(unsigned int i=0; ibool_item_num=(unsigned int)M2.size(); - matcher->bool_items =new unsigned long long[M2.size()]; - matcher->mapped_ptr =new unsigned int[M2.size()+1]; - matcher->mapped_ids =new unsigned int[matcher->bool_expr_num]; - mem_bytes+=((unsigned int)M2.size()+1+matcher->bool_expr_num)*sizeof(unsigned int)+(unsigned int)M2.size()*sizeof(unsigned long long); - - matcher->mapped_ptr[0]=0; - map< unsigned long long, vector >::const_iterator it=M2.begin(); - for(unsigned int k=0; kbool_items[k]=it->first; - copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]); - matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size(); - } - - M1.clear(); - M2.clear(); - - matcher->bitmap_size=(1U<<27); - matcher->bitmap=new unsigned char[(matcher->bitmap_size)>>3]; - mem_bytes+=(matcher->bitmap_size)>>3; - memset(matcher->bitmap, 0, (matcher->bitmap_size)>>3); - - for(unsigned int i=0; ibool_item_num; i++) - { - unsigned int j=matcher->bool_items[i]&(matcher->bitmap_size-1); - matcher->bitmap[j>>3]|=(1U<<(j&7)); - } - - if(mem_size!=NULL) *mem_size=mem_bytes; - return matcher; -} - -int res_comp(const void * lhs, const void * rhs) -{ - bool_expr_match * _lhs=(bool_expr_match *)lhs; - bool_expr_match * _rhs=(bool_expr_match *)rhs; - return (_lhs->expr_id<_rhs->expr_id) ? 1 : -1; -} - -int do_match(struct bool_expr_item * expr, unsigned long long * item_ids, size_t item_num) -{ - unsigned int i=0; - for(unsigned int j=0; jitem_num; ++j) - { - if(expr->items[j].not_flag==0) - { - while(iitems[j].item_id) ++i; - if(i==item_num || item_ids[i]>expr->items[j].item_id) return 0; - ++i; - } - else - { - while(iitems[j].item_id) ++i; - if(iitems[j].item_id) return 0; - } - } - - return 1; -} - -int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result) -{ - if(matcher==NULL) return -1; - if(item_num==0) return 0; - -// sort(item_ids, item_ids+item_num); -// size_t J=0; -// for(unsigned int i=1; ibitmap_size-1); - if((matcher->bitmap[t>>3]&(1U<<(t&7)))==0) continue; - - int l=0, h=(int)matcher->bool_item_num-1; - while(l<=h) - { - int m=(l+h)/2; - if(item_ids[i]==matcher->bool_items[m]) - { - for(unsigned int j=matcher->mapped_ptr[m]; jmapped_ptr[m+1]; j++) - { - unsigned int idx=matcher->mapped_ids[j]; - int ret=do_match(matcher->bool_expr_items+idx, item_ids, item_num); - if(ret==1) - { - if(r==n_result) goto END; - results[r++]=matcher->bool_expr_ids[idx]; - } - } - break; - } - else if(item_ids[i]bool_items[m]) - { - h=m-1; - } - else - { - l=m+1; - } - } - } - -END: - qsort(results, r, sizeof(bool_expr_match), res_comp); - return r; -} - -void bool_matcher_free(struct bool_matcher * matcher) -{ - if(matcher==NULL) return; - - delete [] matcher->bool_expr_ids; - for(unsigned int i=0; ibool_expr_num; i++) delete [] matcher->bool_expr_items[i].items; - delete [] matcher->bool_expr_items; - - delete [] matcher->bool_items; - delete [] matcher->mapped_ptr; - delete [] matcher->mapped_ids; - delete [] matcher->bitmap; - delete matcher; - return; -} diff --git a/src/entry/cJSON.c b/src/entry/cJSON.c deleted file mode 100644 index cbdec41..0000000 --- a/src/entry/cJSON.c +++ /dev/null @@ -1,2932 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -/* cJSON */ -/* JSON parser in C. */ - -/* disable warnings about old C89 functions in MSVC */ -#if !defined(_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) -#define _CRT_SECURE_NO_DEPRECATE -#endif - -#ifdef __GNUC__ -#pragma GCC visibility push(default) -#endif -#if defined(_MSC_VER) -#pragma warning (push) -/* disable warning about single line comments in system headers */ -#pragma warning (disable : 4001) -#endif - -#include -#include -#include -#include -#include -#include - -#ifdef ENABLE_LOCALES -#include -#endif - -#if defined(_MSC_VER) -#pragma warning (pop) -#endif -#ifdef __GNUC__ -#pragma GCC visibility pop -#endif - -#include "cJSON.h" - -/* define our own boolean type */ -#define true ((cJSON_bool)1) -#define false ((cJSON_bool)0) - -typedef struct { - const unsigned char *json; - size_t position; -} error; -static error global_error = { NULL, 0 }; - -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) -{ - return (const char*) (global_error.json + global_error.position); -} - -CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item) { - if (!cJSON_IsString(item)) { - return NULL; - } - - return item->valuestring; -} - -/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ -#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 7) - #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. -#endif - -CJSON_PUBLIC(const char*) cJSON_Version(void) -{ - static char version[15]; - sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); - - return version; -} - -/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ -static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) -{ - if ((string1 == NULL) || (string2 == NULL)) - { - return 1; - } - - if (string1 == string2) - { - return 0; - } - - for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) - { - if (*string1 == '\0') - { - return 0; - } - } - - return tolower(*string1) - tolower(*string2); -} - -typedef struct internal_hooks -{ - void *(*allocate)(size_t size); - void (*deallocate)(void *pointer); - void *(*reallocate)(void *pointer, size_t size); -} internal_hooks; - -#if defined(_MSC_VER) -/* work around MSVC error C2322: '...' address of dillimport '...' is not static */ -static void *internal_malloc(size_t size) -{ - return malloc(size); -} -static void internal_free(void *pointer) -{ - free(pointer); -} -static void *internal_realloc(void *pointer, size_t size) -{ - return realloc(pointer, size); -} -#else -#define internal_malloc malloc -#define internal_free free -#define internal_realloc realloc -#endif - -static internal_hooks global_hooks = { internal_malloc, internal_free, internal_realloc }; - -static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) -{ - size_t length = 0; - unsigned char *copy = NULL; - - if (string == NULL) - { - return NULL; - } - - length = strlen((const char*)string) + sizeof(""); - copy = (unsigned char*)hooks->allocate(length); - if (copy == NULL) - { - return NULL; - } - memcpy(copy, string, length); - - return copy; -} - -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) -{ - if (hooks == NULL) - { - /* Reset hooks */ - global_hooks.allocate = malloc; - global_hooks.deallocate = free; - global_hooks.reallocate = realloc; - return; - } - - global_hooks.allocate = malloc; - if (hooks->malloc_fn != NULL) - { - global_hooks.allocate = hooks->malloc_fn; - } - - global_hooks.deallocate = free; - if (hooks->free_fn != NULL) - { - global_hooks.deallocate = hooks->free_fn; - } - - /* use realloc only if both free and malloc are used */ - global_hooks.reallocate = NULL; - if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) - { - global_hooks.reallocate = realloc; - } -} - -/* Internal constructor. */ -static cJSON *cJSON_New_Item(const internal_hooks * const hooks) -{ - cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); - if (node) - { - memset(node, '\0', sizeof(cJSON)); - } - - return node; -} - -/* Delete a cJSON structure. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) -{ - cJSON *next = NULL; - while (item != NULL) - { - next = item->next; - if (!(item->type & cJSON_IsReference) && (item->child != NULL)) - { - cJSON_Delete(item->child); - } - if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) - { - global_hooks.deallocate(item->valuestring); - } - if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) - { - global_hooks.deallocate(item->string); - } - global_hooks.deallocate(item); - item = next; - } -} - -/* get the decimal point character of the current locale */ -static unsigned char get_decimal_point(void) -{ -#ifdef ENABLE_LOCALES - struct lconv *lconv = localeconv(); - return (unsigned char) lconv->decimal_point[0]; -#else - return '.'; -#endif -} - -typedef struct -{ - const unsigned char *content; - size_t length; - size_t offset; - size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ - internal_hooks hooks; -} parse_buffer; - -/* check if the given size is left to read in a given parse buffer (starting with 1) */ -#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) -/* check if the buffer can be accessed at the given index (starting with 0) */ -#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) -#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) -/* get a pointer to the buffer at the position */ -#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) - -/* Parse the input text to generate a number, and populate the result into item. */ -static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) -{ - double number = 0; - unsigned char *after_end = NULL; - unsigned char number_c_string[64]; - unsigned char decimal_point = get_decimal_point(); - size_t i = 0; - - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; - } - - /* copy the number into a temporary buffer and replace '.' with the decimal point - * of the current locale (for strtod) - * This also takes care of '\0' not necessarily being available for marking the end of the input */ - for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++) - { - switch (buffer_at_offset(input_buffer)[i]) - { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '+': - case '-': - case 'e': - case 'E': - number_c_string[i] = buffer_at_offset(input_buffer)[i]; - break; - - case '.': - number_c_string[i] = decimal_point; - break; - - default: - goto loop_end; - } - } -loop_end: - number_c_string[i] = '\0'; - - number = strtod((const char*)number_c_string, (char**)&after_end); - if (number_c_string == after_end) - { - return false; /* parse_error */ - } - - item->valuedouble = number; - - /* use saturation in case of overflow */ - if (number >= INT_MAX) - { - item->valueint = INT_MAX; - } - else if (number <= INT_MIN) - { - item->valueint = INT_MIN; - } - else - { - item->valueint = (int)number; - } - - item->type = cJSON_Number; - - input_buffer->offset += (size_t)(after_end - number_c_string); - return true; -} - -/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) -{ - if (number >= INT_MAX) - { - object->valueint = INT_MAX; - } - else if (number <= INT_MIN) - { - object->valueint = INT_MIN; - } - else - { - object->valueint = (int)number; - } - - return object->valuedouble = number; -} - -typedef struct -{ - unsigned char *buffer; - size_t length; - size_t offset; - size_t depth; /* current nesting depth (for formatted printing) */ - cJSON_bool noalloc; - cJSON_bool format; /* is this print a formatted print */ - internal_hooks hooks; -} printbuffer; - -/* realloc printbuffer if necessary to have at least "needed" bytes more */ -static unsigned char* ensure(printbuffer * const p, size_t needed) -{ - unsigned char *newbuffer = NULL; - size_t newsize = 0; - - if ((p == NULL) || (p->buffer == NULL)) - { - return NULL; - } - - if ((p->length > 0) && (p->offset >= p->length)) - { - /* make sure that offset is valid */ - return NULL; - } - - if (needed > INT_MAX) - { - /* sizes bigger than INT_MAX are currently not supported */ - return NULL; - } - - needed += p->offset + 1; - if (needed <= p->length) - { - return p->buffer + p->offset; - } - - if (p->noalloc) { - return NULL; - } - - /* calculate new buffer size */ - if (needed > (INT_MAX / 2)) - { - /* overflow of int, use INT_MAX if possible */ - if (needed <= INT_MAX) - { - newsize = INT_MAX; - } - else - { - return NULL; - } - } - else - { - newsize = needed * 2; - } - - if (p->hooks.reallocate != NULL) - { - /* reallocate with realloc if available */ - newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); - if (newbuffer == NULL) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - } - else - { - /* otherwise reallocate manually */ - newbuffer = (unsigned char*)p->hooks.allocate(newsize); - if (!newbuffer) - { - p->hooks.deallocate(p->buffer); - p->length = 0; - p->buffer = NULL; - - return NULL; - } - if (newbuffer) - { - memcpy(newbuffer, p->buffer, p->offset + 1); - } - p->hooks.deallocate(p->buffer); - } - p->length = newsize; - p->buffer = newbuffer; - - return newbuffer + p->offset; -} - -/* calculate the new length of the string in a printbuffer and update the offset */ -static void update_offset(printbuffer * const buffer) -{ - const unsigned char *buffer_pointer = NULL; - if ((buffer == NULL) || (buffer->buffer == NULL)) - { - return; - } - buffer_pointer = buffer->buffer + buffer->offset; - - buffer->offset += strlen((const char*)buffer_pointer); -} - -/* Render the number nicely from the given item into a string. */ -static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - double d = item->valuedouble; - int length = 0; - size_t i = 0; - unsigned char number_buffer[26]; /* temporary buffer to print the number into */ - unsigned char decimal_point = get_decimal_point(); - double test; - - if (output_buffer == NULL) - { - return false; - } - - /* This checks for NaN and Infinity */ - if ((d * 0) != 0) - { - length = sprintf((char*)number_buffer, "null"); - } - else - { - /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ - length = sprintf((char*)number_buffer, "%1.15g", d); - - /* Check whether the original double can be recovered */ - if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || ((double)test != d)) - { - /* If not, print with 17 decimal places of precision */ - length = sprintf((char*)number_buffer, "%1.17g", d); - } - } - - /* sprintf failed or buffer overrun occured */ - if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) - { - return false; - } - - /* reserve appropriate space in the output */ - output_pointer = ensure(output_buffer, (size_t)length + sizeof("")); - if (output_pointer == NULL) - { - return false; - } - - /* copy the printed number to the output and replace locale - * dependent decimal point with '.' */ - for (i = 0; i < ((size_t)length); i++) - { - if (number_buffer[i] == decimal_point) - { - output_pointer[i] = '.'; - continue; - } - - output_pointer[i] = number_buffer[i]; - } - output_pointer[i] = '\0'; - - output_buffer->offset += (size_t)length; - - return true; -} - -/* parse 4 digit hexadecimal number */ -static unsigned parse_hex4(const unsigned char * const input) -{ - unsigned int h = 0; - size_t i = 0; - - for (i = 0; i < 4; i++) - { - /* parse digit */ - if ((input[i] >= '0') && (input[i] <= '9')) - { - h += (unsigned int) input[i] - '0'; - } - else if ((input[i] >= 'A') && (input[i] <= 'F')) - { - h += (unsigned int) 10 + input[i] - 'A'; - } - else if ((input[i] >= 'a') && (input[i] <= 'f')) - { - h += (unsigned int) 10 + input[i] - 'a'; - } - else /* invalid */ - { - return 0; - } - - if (i < 3) - { - /* shift left to make place for the next nibble */ - h = h << 4; - } - } - - return h; -} - -/* converts a UTF-16 literal to UTF-8 - * A literal can be one or two sequences of the form \uXXXX */ -static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) -{ - long unsigned int codepoint = 0; - unsigned int first_code = 0; - const unsigned char *first_sequence = input_pointer; - unsigned char utf8_length = 0; - unsigned char utf8_position = 0; - unsigned char sequence_length = 0; - unsigned char first_byte_mark = 0; - - if ((input_end - first_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - /* get the first utf16 sequence */ - first_code = parse_hex4(first_sequence + 2); - - /* check that the code is valid */ - if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) - { - goto fail; - } - - /* UTF16 surrogate pair */ - if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) - { - const unsigned char *second_sequence = first_sequence + 6; - unsigned int second_code = 0; - sequence_length = 12; /* \uXXXX\uXXXX */ - - if ((input_end - second_sequence) < 6) - { - /* input ends unexpectedly */ - goto fail; - } - - if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) - { - /* missing second half of the surrogate pair */ - goto fail; - } - - /* get the second utf16 sequence */ - second_code = parse_hex4(second_sequence + 2); - /* check that the code is valid */ - if ((second_code < 0xDC00) || (second_code > 0xDFFF)) - { - /* invalid second half of the surrogate pair */ - goto fail; - } - - - /* calculate the unicode codepoint from the surrogate pair */ - codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); - } - else - { - sequence_length = 6; /* \uXXXX */ - codepoint = first_code; - } - - /* encode as UTF-8 - * takes at maximum 4 bytes to encode: - * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ - if (codepoint < 0x80) - { - /* normal ascii, encoding 0xxxxxxx */ - utf8_length = 1; - } - else if (codepoint < 0x800) - { - /* two bytes, encoding 110xxxxx 10xxxxxx */ - utf8_length = 2; - first_byte_mark = 0xC0; /* 11000000 */ - } - else if (codepoint < 0x10000) - { - /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ - utf8_length = 3; - first_byte_mark = 0xE0; /* 11100000 */ - } - else if (codepoint <= 0x10FFFF) - { - /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ - utf8_length = 4; - first_byte_mark = 0xF0; /* 11110000 */ - } - else - { - /* invalid unicode codepoint */ - goto fail; - } - - /* encode as utf8 */ - for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) - { - /* 10xxxxxx */ - (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); - codepoint >>= 6; - } - /* encode first byte */ - if (utf8_length > 1) - { - (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); - } - else - { - (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); - } - - *output_pointer += utf8_length; - - return sequence_length; - -fail: - return 0; -} - -/* Parse the input text into an unescaped cinput, and populate item. */ -static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) -{ - const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; - const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; - unsigned char *output_pointer = NULL; - unsigned char *output = NULL; - - /* not a string */ - if (buffer_at_offset(input_buffer)[0] != '\"') - { - goto fail; - } - - { - /* calculate approximate size of the output (overestimate) */ - size_t allocation_length = 0; - size_t skipped_bytes = 0; - while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) - { - /* is escape sequence */ - if (input_end[0] == '\\') - { - if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) - { - /* prevent buffer overflow when last input character is a backslash */ - goto fail; - } - skipped_bytes++; - input_end++; - } - input_end++; - } - if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) - { - goto fail; /* string ended unexpectedly */ - } - - /* This is at most how much we need for the output */ - allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; - output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); - if (output == NULL) - { - goto fail; /* allocation failure */ - } - } - - output_pointer = output; - /* loop through the string literal */ - while (input_pointer < input_end) - { - if (*input_pointer != '\\') - { - *output_pointer++ = *input_pointer++; - } - /* escape sequence */ - else - { - unsigned char sequence_length = 2; - if ((input_end - input_pointer) < 1) - { - goto fail; - } - - switch (input_pointer[1]) - { - case 'b': - *output_pointer++ = '\b'; - break; - case 'f': - *output_pointer++ = '\f'; - break; - case 'n': - *output_pointer++ = '\n'; - break; - case 'r': - *output_pointer++ = '\r'; - break; - case 't': - *output_pointer++ = '\t'; - break; - case '\"': - case '\\': - case '/': - *output_pointer++ = input_pointer[1]; - break; - - /* UTF-16 literal */ - case 'u': - sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); - if (sequence_length == 0) - { - /* failed to convert UTF16-literal to UTF-8 */ - goto fail; - } - break; - - default: - goto fail; - } - input_pointer += sequence_length; - } - } - - /* zero terminate the output */ - *output_pointer = '\0'; - - item->type = cJSON_String; - item->valuestring = (char*)output; - - input_buffer->offset = (size_t) (input_end - input_buffer->content); - input_buffer->offset++; - - return true; - -fail: - if (output != NULL) - { - input_buffer->hooks.deallocate(output); - } - - if (input_pointer != NULL) - { - input_buffer->offset = (size_t)(input_pointer - input_buffer->content); - } - - return false; -} - -/* Render the cstring provided to an escaped version that can be printed. */ -static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) -{ - const unsigned char *input_pointer = NULL; - unsigned char *output = NULL; - unsigned char *output_pointer = NULL; - size_t output_length = 0; - /* numbers of additional characters needed for escaping */ - size_t escape_characters = 0; - - if (output_buffer == NULL) - { - return false; - } - - /* empty string */ - if (input == NULL) - { - output = ensure(output_buffer, sizeof("\"\"")); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "\"\""); - - return true; - } - - /* set "flag" to 1 if something needs to be escaped */ - for (input_pointer = input; *input_pointer; input_pointer++) - { - switch (*input_pointer) - { - case '\"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - /* one character escape sequence */ - escape_characters++; - break; - default: - if (*input_pointer < 32) - { - /* UTF-16 escape sequence uXXXX */ - escape_characters += 5; - } - break; - } - } - output_length = (size_t)(input_pointer - input) + escape_characters; - - output = ensure(output_buffer, output_length + sizeof("\"\"")); - if (output == NULL) - { - return false; - } - - /* no characters have to be escaped */ - if (escape_characters == 0) - { - output[0] = '\"'; - memcpy(output + 1, input, output_length); - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; - } - - output[0] = '\"'; - output_pointer = output + 1; - /* copy the string */ - for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) - { - if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) - { - /* normal character, copy */ - *output_pointer = *input_pointer; - } - else - { - /* character needs to be escaped */ - *output_pointer++ = '\\'; - switch (*input_pointer) - { - case '\\': - *output_pointer = '\\'; - break; - case '\"': - *output_pointer = '\"'; - break; - case '\b': - *output_pointer = 'b'; - break; - case '\f': - *output_pointer = 'f'; - break; - case '\n': - *output_pointer = 'n'; - break; - case '\r': - *output_pointer = 'r'; - break; - case '\t': - *output_pointer = 't'; - break; - default: - /* escape and print as unicode codepoint */ - sprintf((char*)output_pointer, "u%04x", *input_pointer); - output_pointer += 4; - break; - } - } - } - output[output_length + 1] = '\"'; - output[output_length + 2] = '\0'; - - return true; -} - -/* Invoke print_string_ptr (which is useful) on an item. */ -static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) -{ - return print_string_ptr((unsigned char*)item->valuestring, p); -} - -/* Predeclare these prototypes. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); - -/* Utility to jump whitespace and cr/lf */ -static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) -{ - if ((buffer == NULL) || (buffer->content == NULL)) - { - return NULL; - } - - while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) - { - buffer->offset++; - } - - if (buffer->offset == buffer->length) - { - buffer->offset--; - } - - return buffer; -} - -/* skip the UTF-8 BOM (byte order mark) if it is at the beginning of a buffer */ -static parse_buffer *skip_utf8_bom(parse_buffer * const buffer) -{ - if ((buffer == NULL) || (buffer->content == NULL) || (buffer->offset != 0)) - { - return NULL; - } - - if (can_access_at_index(buffer, 4) && (strncmp((const char*)buffer_at_offset(buffer), "\xEF\xBB\xBF", 3) == 0)) - { - buffer->offset += 3; - } - - return buffer; -} - -/* Parse an object - create a new root, and populate. */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) -{ - parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; - cJSON *item = NULL; - - /* reset error position */ - global_error.json = NULL; - global_error.position = 0; - - if (value == NULL) - { - goto fail; - } - - buffer.content = (const unsigned char*)value; - buffer.length = strlen((const char*)value) + sizeof(""); - buffer.offset = 0; - buffer.hooks = global_hooks; - - item = cJSON_New_Item(&global_hooks); - if (item == NULL) /* memory fail */ - { - goto fail; - } - - if (!parse_value(item, buffer_skip_whitespace(skip_utf8_bom(&buffer)))) - { - /* parse failure. ep is set. */ - goto fail; - } - - /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ - if (require_null_terminated) - { - buffer_skip_whitespace(&buffer); - if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') - { - goto fail; - } - } - if (return_parse_end) - { - *return_parse_end = (const char*)buffer_at_offset(&buffer); - } - - return item; - -fail: - if (item != NULL) - { - cJSON_Delete(item); - } - - if (value != NULL) - { - error local_error; - local_error.json = (const unsigned char*)value; - local_error.position = 0; - - if (buffer.offset < buffer.length) - { - local_error.position = buffer.offset; - } - else if (buffer.length > 0) - { - local_error.position = buffer.length - 1; - } - - if (return_parse_end != NULL) - { - *return_parse_end = (const char*)local_error.json + local_error.position; - } - - global_error = local_error; - } - - return NULL; -} - -/* Default options for cJSON_Parse */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) -{ - return cJSON_ParseWithOpts(value, 0, 0); -} - -#define cjson_min(a, b) ((a < b) ? a : b) - -static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) -{ - static const size_t default_buffer_size = 256; - printbuffer buffer[1]; - unsigned char *printed = NULL; - - memset(buffer, 0, sizeof(buffer)); - - /* create buffer */ - buffer->buffer = (unsigned char*) hooks->allocate(default_buffer_size); - buffer->length = default_buffer_size; - buffer->format = format; - buffer->hooks = *hooks; - if (buffer->buffer == NULL) - { - goto fail; - } - - /* print the value */ - if (!print_value(item, buffer)) - { - goto fail; - } - update_offset(buffer); - - /* check if reallocate is available */ - if (hooks->reallocate != NULL) - { - printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1); - if (printed == NULL) { - goto fail; - } - buffer->buffer = NULL; - } - else /* otherwise copy the JSON over to a new buffer */ - { - printed = (unsigned char*) hooks->allocate(buffer->offset + 1); - if (printed == NULL) - { - goto fail; - } - memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); - printed[buffer->offset] = '\0'; /* just to be sure */ - - /* free the buffer */ - hooks->deallocate(buffer->buffer); - } - - return printed; - -fail: - if (buffer->buffer != NULL) - { - hooks->deallocate(buffer->buffer); - } - - if (printed != NULL) - { - hooks->deallocate(printed); - } - - return NULL; -} - -/* Render a cJSON item/entity/structure to text. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) -{ - return (char*)print(item, true, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) -{ - return (char*)print(item, false, &global_hooks); -} - -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if (prebuffer < 0) - { - return NULL; - } - - p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); - if (!p.buffer) - { - return NULL; - } - - p.length = (size_t)prebuffer; - p.offset = 0; - p.noalloc = false; - p.format = fmt; - p.hooks = global_hooks; - - if (!print_value(item, &p)) - { - global_hooks.deallocate(p.buffer); - return NULL; - } - - return (char*)p.buffer; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buf, const int len, const cJSON_bool fmt) -{ - printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; - - if ((len < 0) || (buf == NULL)) - { - return false; - } - - p.buffer = (unsigned char*)buf; - p.length = (size_t)len; - p.offset = 0; - p.noalloc = true; - p.format = fmt; - p.hooks = global_hooks; - - return print_value(item, &p); -} - -/* Parser core - when encountering text, process appropriately. */ -static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) -{ - if ((input_buffer == NULL) || (input_buffer->content == NULL)) - { - return false; /* no input */ - } - - /* parse the different types of values */ - /* null */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) - { - item->type = cJSON_NULL; - input_buffer->offset += 4; - return true; - } - /* false */ - if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) - { - item->type = cJSON_False; - input_buffer->offset += 5; - return true; - } - /* true */ - if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) - { - item->type = cJSON_True; - item->valueint = 1; - input_buffer->offset += 4; - return true; - } - /* string */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) - { - return parse_string(item, input_buffer); - } - /* number */ - if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) - { - return parse_number(item, input_buffer); - } - /* array */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) - { - return parse_array(item, input_buffer); - } - /* object */ - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) - { - return parse_object(item, input_buffer); - } - - return false; -} - -/* Render a value to text. */ -static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output = NULL; - - if ((item == NULL) || (output_buffer == NULL)) - { - return false; - } - - switch ((item->type) & 0xFF) - { - case cJSON_NULL: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "null"); - return true; - - case cJSON_False: - output = ensure(output_buffer, 6); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "false"); - return true; - - case cJSON_True: - output = ensure(output_buffer, 5); - if (output == NULL) - { - return false; - } - strcpy((char*)output, "true"); - return true; - - case cJSON_Number: - return print_number(item, output_buffer); - - case cJSON_Raw: - { - size_t raw_length = 0; - if (item->valuestring == NULL) - { - return false; - } - - raw_length = strlen(item->valuestring) + sizeof(""); - output = ensure(output_buffer, raw_length); - if (output == NULL) - { - return false; - } - memcpy(output, item->valuestring, raw_length); - return true; - } - - case cJSON_String: - return print_string(item, output_buffer); - - case cJSON_Array: - return print_array(item, output_buffer); - - case cJSON_Object: - return print_object(item, output_buffer); - - default: - return false; - } -} - -/* Build an array from input text. */ -static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* head of the linked list */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (buffer_at_offset(input_buffer)[0] != '[') - { - /* not an array */ - goto fail; - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) - { - /* empty array */ - goto success; - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse next value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') - { - goto fail; /* expected end of array */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Array; - item->child = head; - - input_buffer->offset++; - - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an array to text */ -static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_element = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output array. */ - /* opening square bracket */ - output_pointer = ensure(output_buffer, 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer = '['; - output_buffer->offset++; - output_buffer->depth++; - - while (current_element != NULL) - { - if (!print_value(current_element, output_buffer)) - { - return false; - } - update_offset(output_buffer); - if (current_element->next) - { - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ','; - if(output_buffer->format) - { - *output_pointer++ = ' '; - } - *output_pointer = '\0'; - output_buffer->offset += length; - } - current_element = current_element->next; - } - - output_pointer = ensure(output_buffer, 2); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ']'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Build an object from the text. */ -static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) -{ - cJSON *head = NULL; /* linked list head */ - cJSON *current_item = NULL; - - if (input_buffer->depth >= CJSON_NESTING_LIMIT) - { - return false; /* to deeply nested */ - } - input_buffer->depth++; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) - { - goto fail; /* not an object */ - } - - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) - { - goto success; /* empty object */ - } - - /* check if we skipped to the end of the buffer */ - if (cannot_access_at_index(input_buffer, 0)) - { - input_buffer->offset--; - goto fail; - } - - /* step back to character in front of the first element */ - input_buffer->offset--; - /* loop through the comma separated array elements */ - do - { - /* allocate next item */ - cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); - if (new_item == NULL) - { - goto fail; /* allocation failure */ - } - - /* attach next item to list */ - if (head == NULL) - { - /* start the linked list */ - current_item = head = new_item; - } - else - { - /* add to the end and advance */ - current_item->next = new_item; - new_item->prev = current_item; - current_item = new_item; - } - - /* parse the name of the child */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_string(current_item, input_buffer)) - { - goto fail; /* faile to parse name */ - } - buffer_skip_whitespace(input_buffer); - - /* swap valuestring and string, because we parsed the name */ - current_item->string = current_item->valuestring; - current_item->valuestring = NULL; - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) - { - goto fail; /* invalid object */ - } - - /* parse the value */ - input_buffer->offset++; - buffer_skip_whitespace(input_buffer); - if (!parse_value(current_item, input_buffer)) - { - goto fail; /* failed to parse value */ - } - buffer_skip_whitespace(input_buffer); - } - while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); - - if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) - { - goto fail; /* expected end of object */ - } - -success: - input_buffer->depth--; - - item->type = cJSON_Object; - item->child = head; - - input_buffer->offset++; - return true; - -fail: - if (head != NULL) - { - cJSON_Delete(head); - } - - return false; -} - -/* Render an object to text. */ -static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) -{ - unsigned char *output_pointer = NULL; - size_t length = 0; - cJSON *current_item = item->child; - - if (output_buffer == NULL) - { - return false; - } - - /* Compose the output: */ - length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - - *output_pointer++ = '{'; - output_buffer->depth++; - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - output_buffer->offset += length; - - while (current_item) - { - if (output_buffer->format) - { - size_t i; - output_pointer = ensure(output_buffer, output_buffer->depth); - if (output_pointer == NULL) - { - return false; - } - for (i = 0; i < output_buffer->depth; i++) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += output_buffer->depth; - } - - /* print key */ - if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - length = (size_t) (output_buffer->format ? 2 : 1); - output_pointer = ensure(output_buffer, length); - if (output_pointer == NULL) - { - return false; - } - *output_pointer++ = ':'; - if (output_buffer->format) - { - *output_pointer++ = '\t'; - } - output_buffer->offset += length; - - /* print value */ - if (!print_value(current_item, output_buffer)) - { - return false; - } - update_offset(output_buffer); - - /* print comma if not last */ - length = (size_t) ((output_buffer->format ? 1 : 0) + (current_item->next ? 1 : 0)); - output_pointer = ensure(output_buffer, length + 1); - if (output_pointer == NULL) - { - return false; - } - if (current_item->next) - { - *output_pointer++ = ','; - } - - if (output_buffer->format) - { - *output_pointer++ = '\n'; - } - *output_pointer = '\0'; - output_buffer->offset += length; - - current_item = current_item->next; - } - - output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); - if (output_pointer == NULL) - { - return false; - } - if (output_buffer->format) - { - size_t i; - for (i = 0; i < (output_buffer->depth - 1); i++) - { - *output_pointer++ = '\t'; - } - } - *output_pointer++ = '}'; - *output_pointer = '\0'; - output_buffer->depth--; - - return true; -} - -/* Get Array size/item / object item. */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) -{ - cJSON *child = NULL; - size_t size = 0; - - if (array == NULL) - { - return 0; - } - - child = array->child; - - while(child != NULL) - { - size++; - child = child->next; - } - - /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ - - return (int)size; -} - -static cJSON* get_array_item(const cJSON *array, size_t index) -{ - cJSON *current_child = NULL; - - if (array == NULL) - { - return NULL; - } - - current_child = array->child; - while ((current_child != NULL) && (index > 0)) - { - index--; - current_child = current_child->next; - } - - return current_child; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) -{ - if (index < 0) - { - return NULL; - } - - return get_array_item(array, (size_t)index); -} - -static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) -{ - cJSON *current_element = NULL; - - if ((object == NULL) || (name == NULL)) - { - return NULL; - } - - current_element = object->child; - if (case_sensitive) - { - while ((current_element != NULL) && (strcmp(name, current_element->string) != 0)) - { - current_element = current_element->next; - } - } - else - { - while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) - { - current_element = current_element->next; - } - } - - return current_element; -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, false); -} - -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) -{ - return get_object_item(object, string, true); -} - -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) -{ - return cJSON_GetObjectItem(object, string) ? 1 : 0; -} - -/* Utility for array list handling. */ -static void suffix_object(cJSON *prev, cJSON *item) -{ - prev->next = item; - item->prev = prev; -} - -/* Utility for handling references. */ -static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) -{ - cJSON *reference = NULL; - if (item == NULL) - { - return NULL; - } - - reference = cJSON_New_Item(hooks); - if (reference == NULL) - { - return NULL; - } - - memcpy(reference, item, sizeof(cJSON)); - reference->string = NULL; - reference->type |= cJSON_IsReference; - reference->next = reference->prev = NULL; - return reference; -} - -static cJSON_bool add_item_to_array(cJSON *array, cJSON *item) -{ - cJSON *child = NULL; - - if ((item == NULL) || (array == NULL)) - { - return false; - } - - child = array->child; - - if (child == NULL) - { - /* list is empty, start new one */ - array->child = item; - } - else - { - /* append to the end */ - while (child->next) - { - child = child->next; - } - suffix_object(child, item); - } - - return true; -} - -/* Add item to array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item) -{ - add_item_to_array(array, item); -} - -#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic push -#endif -#ifdef __GNUC__ -#pragma GCC diagnostic ignored "-Wcast-qual" -#endif -/* helper function to cast away const */ -static void* cast_away_const(const void* string) -{ - return (void*)string; -} -#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) - #pragma GCC diagnostic pop -#endif - - -static cJSON_bool add_item_to_object(cJSON * const object, const char * const string, cJSON * const item, const internal_hooks * const hooks, const cJSON_bool constant_key) -{ - char *new_key = NULL; - int new_type = cJSON_Invalid; - - if ((object == NULL) || (string == NULL) || (item == NULL)) - { - return false; - } - - if (constant_key) - { - new_key = (char*)cast_away_const(string); - new_type = item->type | cJSON_StringIsConst; - } - else - { - new_key = (char*)cJSON_strdup((const unsigned char*)string, hooks); - if (new_key == NULL) - { - return false; - } - - new_type = item->type & ~cJSON_StringIsConst; - } - - if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) - { - hooks->deallocate(item->string); - } - - item->string = new_key; - item->type = new_type; - - return add_item_to_array(object, item); -} - -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) -{ - add_item_to_object(object, string, item, &global_hooks, false); -} - -/* Add an item to an object with constant string as key */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) -{ - add_item_to_object(object, string, item, &global_hooks, true); -} - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) -{ - if (array == NULL) - { - return; - } - - add_item_to_array(array, create_reference(item, &global_hooks)); -} - -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) -{ - if ((object == NULL) || (string == NULL)) - { - return; - } - - add_item_to_object(object, string, create_reference(item, &global_hooks), &global_hooks, false); -} - -CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name) -{ - cJSON *null = cJSON_CreateNull(); - if (add_item_to_object(object, name, null, &global_hooks, false)) - { - return null; - } - - cJSON_Delete(null); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name) -{ - cJSON *true_item = cJSON_CreateTrue(); - if (add_item_to_object(object, name, true_item, &global_hooks, false)) - { - return true_item; - } - - cJSON_Delete(true_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name) -{ - cJSON *false_item = cJSON_CreateFalse(); - if (add_item_to_object(object, name, false_item, &global_hooks, false)) - { - return false_item; - } - - cJSON_Delete(false_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean) -{ - cJSON *bool_item = cJSON_CreateBool(boolean); - if (add_item_to_object(object, name, bool_item, &global_hooks, false)) - { - return bool_item; - } - - cJSON_Delete(bool_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number) -{ - cJSON *number_item = cJSON_CreateNumber(number); - if (add_item_to_object(object, name, number_item, &global_hooks, false)) - { - return number_item; - } - - cJSON_Delete(number_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string) -{ - cJSON *string_item = cJSON_CreateString(string); - if (add_item_to_object(object, name, string_item, &global_hooks, false)) - { - return string_item; - } - - cJSON_Delete(string_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw) -{ - cJSON *raw_item = cJSON_CreateRaw(raw); - if (add_item_to_object(object, name, raw_item, &global_hooks, false)) - { - return raw_item; - } - - cJSON_Delete(raw_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name) -{ - cJSON *object_item = cJSON_CreateObject(); - if (add_item_to_object(object, name, object_item, &global_hooks, false)) - { - return object_item; - } - - cJSON_Delete(object_item); - return NULL; -} - -CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name) -{ - cJSON *array = cJSON_CreateArray(); - if (add_item_to_object(object, name, array, &global_hooks, false)) - { - return array; - } - - cJSON_Delete(array); - return NULL; -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) -{ - if ((parent == NULL) || (item == NULL)) - { - return NULL; - } - - if (item->prev != NULL) - { - /* not the first element */ - item->prev->next = item->next; - } - if (item->next != NULL) - { - /* not the last element */ - item->next->prev = item->prev; - } - - if (item == parent->child) - { - /* first element */ - parent->child = item->next; - } - /* make sure the detached item doesn't point anywhere anymore */ - item->prev = NULL; - item->next = NULL; - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) -{ - if (which < 0) - { - return NULL; - } - - return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) -{ - cJSON_Delete(cJSON_DetachItemFromArray(array, which)); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItem(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); - - return cJSON_DetachItemViaPointer(object, to_detach); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObject(object, string)); -} - -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) -{ - cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); -} - -/* Replace array/object items with new ones. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) -{ - cJSON *after_inserted = NULL; - - if (which < 0) - { - return; - } - - after_inserted = get_array_item(array, (size_t)which); - if (after_inserted == NULL) - { - add_item_to_array(array, newitem); - return; - } - - newitem->next = after_inserted; - newitem->prev = after_inserted->prev; - after_inserted->prev = newitem; - if (after_inserted == array->child) - { - array->child = newitem; - } - else - { - newitem->prev->next = newitem; - } -} - -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) -{ - if ((parent == NULL) || (replacement == NULL) || (item == NULL)) - { - return false; - } - - if (replacement == item) - { - return true; - } - - replacement->next = item->next; - replacement->prev = item->prev; - - if (replacement->next != NULL) - { - replacement->next->prev = replacement; - } - if (replacement->prev != NULL) - { - replacement->prev->next = replacement; - } - if (parent->child == item) - { - parent->child = replacement; - } - - item->next = NULL; - item->prev = NULL; - cJSON_Delete(item); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) -{ - if (which < 0) - { - return; - } - - cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); -} - -static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) -{ - if ((replacement == NULL) || (string == NULL)) - { - return false; - } - - /* replace the name in the replacement */ - if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) - { - cJSON_free(replacement->string); - } - replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - replacement->type &= ~cJSON_StringIsConst; - - cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); - - return true; -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, false); -} - -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) -{ - replace_item_in_object(object, string, newitem, true); -} - -/* Create basic types: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_NULL; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_True; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool b) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = b ? cJSON_True : cJSON_False; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Number; - item->valuedouble = num; - - /* use saturation in case of overflow */ - if (num >= INT_MAX) - { - item->valueint = INT_MAX; - } - else if (num <= INT_MIN) - { - item->valueint = INT_MIN; - } - else - { - item->valueint = (int)num; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_String; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) - { - item->type = cJSON_String | cJSON_IsReference; - item->valuestring = (char*)cast_away_const(string); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) { - item->type = cJSON_Object | cJSON_IsReference; - item->child = (cJSON*)cast_away_const(child); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child) { - cJSON *item = cJSON_New_Item(&global_hooks); - if (item != NULL) { - item->type = cJSON_Array | cJSON_IsReference; - item->child = (cJSON*)cast_away_const(child); - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type = cJSON_Raw; - item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); - if(!item->valuestring) - { - cJSON_Delete(item); - return NULL; - } - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if(item) - { - item->type=cJSON_Array; - } - - return item; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) -{ - cJSON *item = cJSON_New_Item(&global_hooks); - if (item) - { - item->type = cJSON_Object; - } - - return item; -} - -/* Create Arrays: */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if (!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber((double)numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (numbers == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for(i = 0;a && (i < (size_t)count); i++) - { - n = cJSON_CreateNumber(numbers[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p, n); - } - p = n; - } - - return a; -} - -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count) -{ - size_t i = 0; - cJSON *n = NULL; - cJSON *p = NULL; - cJSON *a = NULL; - - if ((count < 0) || (strings == NULL)) - { - return NULL; - } - - a = cJSON_CreateArray(); - - for (i = 0; a && (i < (size_t)count); i++) - { - n = cJSON_CreateString(strings[i]); - if(!n) - { - cJSON_Delete(a); - return NULL; - } - if(!i) - { - a->child = n; - } - else - { - suffix_object(p,n); - } - p = n; - } - - return a; -} - -/* Duplication */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) -{ - cJSON *newitem = NULL; - cJSON *child = NULL; - cJSON *next = NULL; - cJSON *newchild = NULL; - - /* Bail on bad ptr */ - if (!item) - { - goto fail; - } - /* Create new item */ - newitem = cJSON_New_Item(&global_hooks); - if (!newitem) - { - goto fail; - } - /* Copy over all vars */ - newitem->type = item->type & (~cJSON_IsReference); - newitem->valueint = item->valueint; - newitem->valuedouble = item->valuedouble; - if (item->valuestring) - { - newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); - if (!newitem->valuestring) - { - goto fail; - } - } - if (item->string) - { - newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); - if (!newitem->string) - { - goto fail; - } - } - /* If non-recursive, then we're done! */ - if (!recurse) - { - return newitem; - } - /* Walk the ->next chain for the child. */ - child = item->child; - while (child != NULL) - { - newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */ - if (!newchild) - { - goto fail; - } - if (next != NULL) - { - /* If newitem->child already set, then crosswire ->prev and ->next and move on */ - next->next = newchild; - newchild->prev = next; - next = newchild; - } - else - { - /* Set newitem->child and move to it */ - newitem->child = newchild; - next = newchild; - } - child = child->next; - } - - return newitem; - -fail: - if (newitem != NULL) - { - cJSON_Delete(newitem); - } - - return NULL; -} - -CJSON_PUBLIC(void) cJSON_Minify(char *json) -{ - unsigned char *into = (unsigned char*)json; - - if (json == NULL) - { - return; - } - - while (*json) - { - if (*json == ' ') - { - json++; - } - else if (*json == '\t') - { - /* Whitespace characters. */ - json++; - } - else if (*json == '\r') - { - json++; - } - else if (*json=='\n') - { - json++; - } - else if ((*json == '/') && (json[1] == '/')) - { - /* double-slash comments, to end of line. */ - while (*json && (*json != '\n')) - { - json++; - } - } - else if ((*json == '/') && (json[1] == '*')) - { - /* multiline comments. */ - while (*json && !((*json == '*') && (json[1] == '/'))) - { - json++; - } - json += 2; - } - else if (*json == '\"') - { - /* string literals, which are \" sensitive. */ - *into++ = (unsigned char)*json++; - while (*json && (*json != '\"')) - { - if (*json == '\\') - { - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - *into++ = (unsigned char)*json++; - } - else - { - /* All other characters. */ - *into++ = (unsigned char)*json++; - } - } - - /* and null-terminate. */ - *into = '\0'; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Invalid; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_False; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xff) == cJSON_True; -} - - -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & (cJSON_True | cJSON_False)) != 0; -} -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_NULL; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Number; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_String; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Array; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Object; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) -{ - if (item == NULL) - { - return false; - } - - return (item->type & 0xFF) == cJSON_Raw; -} - -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) -{ - if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF)) || cJSON_IsInvalid(a)) - { - return false; - } - - /* check if type is valid */ - switch (a->type & 0xFF) - { - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - case cJSON_Number: - case cJSON_String: - case cJSON_Raw: - case cJSON_Array: - case cJSON_Object: - break; - - default: - return false; - } - - /* identical objects are equal */ - if (a == b) - { - return true; - } - - switch (a->type & 0xFF) - { - /* in these cases and equal type is enough */ - case cJSON_False: - case cJSON_True: - case cJSON_NULL: - return true; - - case cJSON_Number: - if (a->valuedouble == b->valuedouble) - { - return true; - } - return false; - - case cJSON_String: - case cJSON_Raw: - if ((a->valuestring == NULL) || (b->valuestring == NULL)) - { - return false; - } - if (strcmp(a->valuestring, b->valuestring) == 0) - { - return true; - } - - return false; - - case cJSON_Array: - { - cJSON *a_element = a->child; - cJSON *b_element = b->child; - - for (; (a_element != NULL) && (b_element != NULL);) - { - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - - a_element = a_element->next; - b_element = b_element->next; - } - - /* one of the arrays is longer than the other */ - if (a_element != b_element) { - return false; - } - - return true; - } - - case cJSON_Object: - { - cJSON *a_element = NULL; - cJSON *b_element = NULL; - cJSON_ArrayForEach(a_element, a) - { - /* TODO This has O(n^2) runtime, which is horrible! */ - b_element = get_object_item(b, a_element->string, case_sensitive); - if (b_element == NULL) - { - return false; - } - - if (!cJSON_Compare(a_element, b_element, case_sensitive)) - { - return false; - } - } - - /* doing this twice, once on a and b to prevent true comparison if a subset of b - * TODO: Do this the proper way, this is just a fix for now */ - cJSON_ArrayForEach(b_element, b) - { - a_element = get_object_item(a, b_element->string, case_sensitive); - if (a_element == NULL) - { - return false; - } - - if (!cJSON_Compare(b_element, a_element, case_sensitive)) - { - return false; - } - } - - return true; - } - - default: - return false; - } -} - -CJSON_PUBLIC(void *) cJSON_malloc(size_t size) -{ - return global_hooks.allocate(size); -} - -CJSON_PUBLIC(void) cJSON_free(void *object) -{ - global_hooks.deallocate(object); -} diff --git a/src/entry/config_monitor.cpp b/src/entry/config_monitor.cpp deleted file mode 100644 index 1a80fff..0000000 --- a/src/entry/config_monitor.cpp +++ /dev/null @@ -1,418 +0,0 @@ -#include "MESA_handle_logger.h" -#include "config_monitor.h" -#include "Maat_utils.h" -#include -#include -#include -#include -#include -#include - -#define module_config_monitor (module_name_str("MAAT_FILE_MONITOR")) - - -#define CM_UPDATE_TYPE_ERR -1 -#define CM_UPDATE_TYPE_NONE 0 - -#define CM_MAX_TABLE_NUM 256 -#define MAX_CONFIG_FN_LEN 256 -#define MAX_CONFIG_LINE (1024*16) - - -//#define USING_DICTATOR 1 -extern "C" void __real_free(void*p); -struct cm_table_info_t -{ - char table_name[MAX_CONFIG_FN_LEN]; - char cfg_path[MAX_CONFIG_FN_LEN]; - int cfg_num; - char encryp_algorithm[MAX_CONFIG_FN_LEN]; -}; -char* read_nxt_line_from_buff(const char* buff, size_t buff_size, size_t* offset, char*line, int line_size) -{ - int this_offset=0; - const char* p; - //search for CRLF, aka CR '\r'(old Mac), LF '\n'(UNIX) or CRLF"\r\n" (Windows) - p=(const char*)memchr(buff+*offset,'\n',buff_size-*offset); - if(p==NULL)// NOT "\n" or "\r\n" - { - p=(const char*)memchr(buff+*offset,'\r',buff_size-*offset); - } - if(p!=NULL)//point to next character - { - p++; - } - else //Treat rest buff has no CRLF as a line. - { - p=buff+buff_size; - } - this_offset=p-(buff+*offset); - memcpy(line,buff+*offset, MIN(this_offset,line_size-1)); - *offset+=this_offset; - line[MIN(this_offset,line_size-1)]='\0'; - return line; -} -//replacement of glibc scandir, to adapt dictator malloc wrap -#define ENLARGE_STEP 1024 -int my_scandir(const char *dir, struct dirent ***namelist, - int(*filter)(const struct dirent *), - int(*compar)(const void *, const void *)) -{ - DIR * od; - int n = 0; - int DIR_ENT_SIZE=ENLARGE_STEP; - struct dirent ** list = NULL; - struct dirent * p; - struct dirent entry,*result; - - if((dir == NULL) || (namelist == NULL)) - return -1; - - od = opendir(dir); - if(od == NULL) - return -1; - - list = (struct dirent **)malloc(DIR_ENT_SIZE*sizeof(struct dirent *)); - - - while(0==readdir_r(od,&entry,&result)) - { - if(result==NULL) - { - break; - } - if( filter && !filter(&entry)) - continue; - - p = (struct dirent *)malloc(sizeof(struct dirent)); - memcpy((void *)p,(void *)(&entry),sizeof(struct dirent)); - list[n] = p; - - n++; - if(n >= DIR_ENT_SIZE) - { - DIR_ENT_SIZE+=ENLARGE_STEP; - list=(struct dirent **)realloc((void*)list,DIR_ENT_SIZE*sizeof(struct dirent *)); - - } - } - - closedir(od); - - *namelist = list; - - if(compar) - qsort((void *)*namelist,n,sizeof(struct dirent *),compar); - - return n; - -} - int filter_fn(const struct dirent * ent) - { -// files in xfs are not DT_REG. -// if(ent->d_type != DT_REG) -// return 0; - - return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0|| - strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); - } -int get_new_idx_path(long long current_version,const char*file_dir,void* logger,char*** idx_path,int*idx_num) -{ - struct dirent **namelist; - int n=0,i=0,sscanf_ret; - - char update_str[32]={0}; - long long latest_ful_version=0,latest_inc_version=0; - long long config_seq=0; - int *inc_file_idx; - int full_file_idx=0,inc_idx_num=0,path_len=0; - - int update_type=CM_UPDATE_TYPE_NONE; - - - n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); - if (n < 0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"scan dir error"); - update_type=CM_UPDATE_TYPE_ERR; - return update_type; - } - inc_file_idx=(int*)calloc(sizeof(int),n); - inc_idx_num=0; - for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) - { - continue; - } - if(strlen(namelist[i]->d_name)>42) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor - ,"config file %s filename too long,should like full_config_index.00000000000000000001" - ,namelist[i]->d_name); - continue; - } - sscanf_ret=sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",update_str,&config_seq); - if(sscanf_ret!=2) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor - ,"config file %s filename error,should like full_config_index.00000000000000000001" - ,namelist[i]->d_name); - continue; - } - if(strncasecmp(update_str,"full",strlen(update_str))==0) - { - if(config_seq>latest_ful_version) - { - latest_ful_version=config_seq; - full_file_idx=i; - } - } - else if(strncasecmp(update_str,"inc",strlen(update_str))==0) - { - if(config_seq>current_version) - { - inc_file_idx[inc_idx_num]=i; - inc_idx_num++; - if(config_seq>latest_inc_version) - { - latest_inc_version=config_seq; - } - - } - } - else - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor - ,"config file %s,not full or inc config" - ,namelist[i]->d_name); - } - } - //full update - if(latest_ful_version>current_version) - { - *idx_path=(char**)malloc(sizeof(char**)); - path_len=strlen(file_dir)+strlen(namelist[full_file_idx]->d_name)+1+1; - (*idx_path)[0]=(char*)malloc(path_len); - snprintf((*idx_path)[0],path_len,"%s/%s",file_dir,namelist[full_file_idx]->d_name); - *idx_num=1; - update_type=CM_UPDATE_TYPE_FULL; - } - //inc update,it's possible that do inc after full update in this function,but we'll process it at next loop. - else if(latest_inc_version>current_version) - { - - *idx_path=(char**)malloc(sizeof(char**)*inc_idx_num); - for(i=0;id_name)+1+1; - (*idx_path)[i]=(char*)malloc(path_len); - snprintf((*idx_path)[i],path_len,"%s/%s",file_dir,namelist[inc_file_idx[i]]->d_name); - - } - *idx_num=inc_idx_num; - update_type=CM_UPDATE_TYPE_INC; - - } - else - { - update_type=CM_UPDATE_TYPE_NONE; - } - free(inc_file_idx); - for(i=0;iencryp_algorithm)>0) - { - if(key==NULL||strlen((const char*)key)==0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); - return -1; - } - ret=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, &file_sz, error_string, sizeof(error_string)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, decrypt %s failed: %s", - index->cfg_path, error_string); - return -1; - } - } - else - { - ret=load_file_to_memory(index->cfg_path, (unsigned char**)&table_file_buff, &file_sz); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, open %s failed: %s", - index->cfg_path, error_string); - return -1; - } - - } - read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); - sscanf(line, "%d\n", &cfg_num); - - if(cfg_num!=index->cfg_num) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path); - return -1; - } - for(i=0;icfg_path,i,cfg_num); - break; - } - if(line[sizeof(line)-1]!='\0') - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor , - "update error,line size more than %u at of file %s:%d", - sizeof(line),index->cfg_path,i); - continue; - } - ret=update(index->table_name,line,u_para); - if(ret<0) - { - break; - } - } - - free(table_file_buff); - - return 0; -} -const char* path2filename(const char*path) -{ - int i=0; - for(i=strlen(path);i>0;i--) - { - if(path[i]=='/') - { - break; - } - } - return path+i+1; -} -void config_monitor_traverse(long long version,const char*idx_dir, - void (*start)(long long, int, void*),//vesion,CM_UPDATE_TYPE_*,u_para - int (*update)(const char*, const char*, void*),//table name ,line ,u_para - void (*finish)(void*),//u_para - void* u_para, - const char* dec_key, - void* logger) - -{ - - int update_type=CM_UPDATE_TYPE_NONE; - long long new_version=0; - char**idx_path_array=NULL; - const char* table_filename=NULL; - char str_not_care[256]={0}; - int idx_num=0,table_num=0,i=0,j=0; - struct cm_table_info_t table_array[CM_MAX_TABLE_NUM]; - memset(table_array,0,sizeof(table_array)); - update_type=get_new_idx_path(version, idx_dir,logger, &idx_path_array, &idx_num); - if(update_type==CM_UPDATE_TYPE_FULL||update_type==CM_UPDATE_TYPE_INC) - { - - for(i=0;i -#include -int dynamic_array_VERSION_20141202=0; -struct dynamic_array_t* dynamic_array_create(long long init_size, long long step) -{ - struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1); - d_array->array=(void**)calloc(sizeof(void*),init_size); - d_array->size=init_size; - d_array->enlarge_step=step; - return d_array; -} -void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)) -{ - int i; - if(free_data!=NULL) - { - for(i=0;isize;i++) - { - free_data(d_array->array[i]); - } - } - free(d_array->array); - free(d_array); -} -void* dynamic_array_read(struct dynamic_array_t* d_array, long long i) -{ - if(isize) - { - return d_array->array[i]; - } - else - { - return NULL; - } -} -void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data) -{ - int new_size=0; - if(isize) - { - d_array->array[i]=data; - } - else - { - new_size=i+d_array->enlarge_step; - d_array->array=(void**)realloc(d_array->array,new_size*sizeof(void*)); - memset(d_array->array+d_array->size,0,(new_size-d_array->size)*sizeof(void*)); - d_array->size=new_size; - d_array->array[i]=data; - return; - } -} diff --git a/src/entry/gram_index_engine.c b/src/entry/gram_index_engine.c deleted file mode 100644 index 3cbaa36..0000000 --- a/src/entry/gram_index_engine.c +++ /dev/null @@ -1,1354 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "gram_index_engine.h" -#include "queue.h" - -#define HTABLE_SIZE 128*1024 -#define GRAM_CNT_MAX 2 -#define GRAM_MAX 128 -#define TOLERENCE_SIZE 0 -#define UNION_INIT_SIZE 1000 -#define BLOCKSIZE_MIN 3 -#define MEM_OCCUPY 1 -#define CNT_MAX 10 -#define GRAM_CNT_THRESHOLD 10 -#define QUERY_LEN_ACCURACY 0.1 -#define HTABLE_NUM 8 -//#define GIE_INPUT_FORMAT_SFH 1 -//#define GIE_INPUT_FORMAT_PLAIN 0 -#define MAX_LENGTH 10000 -#define KEY_MAX_LENGTH 10 -#define EDIT_DISTN_INSERT_COST 1 -#define EDIT_DISTN_REMOVE_COST 1 -#define EDIT_DISTN_REPLACE_COST 2 -#define MIN(x,y) ((x)<(y)?(x):(y)) - -int before(unsigned int off1, unsigned int off2) -{ - return (signed int)(off1-off2)<0; -} -#define after(off2,off1) before(off1,off2) - -typedef struct -{ - unsigned int user_gram_value; - unsigned int user_position_accuracy; - short ED_reexamine; - short input_format; - MESA_htable_handle id_table; - MESA_htable_handle index_table[HTABLE_NUM]; - unsigned long long mem_occupy; - unsigned long long hash_cnt; -}GIE_handle_inner_t; - - -struct linklist_node -{ - short * position; - struct id_table_data * basicinfo; - short size; - short index; - unsigned long long blocksize; - TAILQ_ENTRY(linklist_node) listentry; -}; - - -struct index_table_data -{ - struct TQ * listhead; - int cnt; -}; - - -struct id_table_data -{ - unsigned int id; - short sfh_length; - short gram_cnt; - unsigned long long blocksize; - char * sfh; - void * tag; - char cfds_lvl; -}; - - -struct htable_handle -{ - MESA_htable_handle runtime_table; - MESA_htable_handle para; -}; - -struct key_list_node -{ - char * key; - int digest_id; - int pos; - unsigned long long blocksize; - TAILQ_ENTRY(key_list_node) keylistentry; -}; - - -unsigned long long hash_cnt; -unsigned long long cnt_sum; - -TAILQ_HEAD(TQ, linklist_node); -TAILQ_HEAD(KL, key_list_node); - -void idtable_free(void * data); -void indextable_free(void * data); -int key_compare(const uchar * key1, uint size1, const uchar * key2, uint size2); -int GIE_insert_indextable(MESA_htable_handle handle, struct id_table_data * info, char * key, unsigned int index,unsigned long long blocksize); - -int GIE_delete_from_indextable_by_key(MESA_htable_handle handle, char * key, unsigned int id); -int GIE_delete(GIE_handle_inner_t * handle, GIE_digest_t * digest); -int GIE_cmp(const void * a, const void * b); -static inline unsigned int get_real_length(const char * string, unsigned int length); -void print_item_iterate(const uchar * key, unsigned int size, void * data, void * user); -inline unsigned long long calc_fh_blocksize(unsigned long long orilen); -static inline unsigned long long get_blocksize_from_head(const char * fuzzy_string, unsigned int str_len); - -MESA_htable_handle copy_htable(void * htable_para,void (* func)(const uchar * key, uint size, void * data, void *user),void (*free_fuc)(void * data)); -void copy_idtable_item_iterate(const uchar * key, uint size, void * data, void * user); -void copy_indextable_item_iterate(const uchar * key, uint size, void * data, void * user); - -GIE_handle_t * GIE_create(const GIE_create_para_t * para) -{ - int i = 0; - GIE_handle_inner_t * handle = (GIE_handle_inner_t *)calloc(1, sizeof(GIE_handle_inner_t)); - handle->mem_occupy = 0; - handle->mem_occupy += sizeof(GIE_handle_inner_t); - - handle->user_gram_value = para->gram_value; - handle->user_position_accuracy = para->position_accuracy; - handle->input_format = para->format; - //handle->user_cmp = GIE_INPUT_FORMAT_PLAIN; - handle->ED_reexamine = para->ED_reexamine; - handle->hash_cnt = 0; - - - MESA_htable_create_args_t idtable_args,indextable_args[HTABLE_NUM]; - memset(&idtable_args, 0, sizeof(idtable_args)); - idtable_args.thread_safe = 0; - idtable_args.hash_slot_size = HTABLE_SIZE; - idtable_args.max_elem_num = 0; - idtable_args.expire_time = 0; - idtable_args.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - idtable_args.key_comp = NULL; - idtable_args.key2index = NULL; - idtable_args.data_free = idtable_free; - idtable_args.data_expire_with_condition = NULL; - idtable_args.recursive = 0; - handle->id_table = MESA_htable_create(&idtable_args, sizeof(idtable_args)); - - for(i = 0;i < HTABLE_NUM;i++) - { - memset(&indextable_args[i], 0, sizeof(indextable_args[i])); - indextable_args[i].thread_safe = 0; - indextable_args[i].hash_slot_size = HTABLE_SIZE; - indextable_args[i].max_elem_num = 0; - indextable_args[i].expire_time = 0; - indextable_args[i].eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - indextable_args[i].key_comp = key_compare; - indextable_args[i].key2index = NULL; - indextable_args[i].data_free = indextable_free; - indextable_args[i].data_expire_with_condition = NULL; - indextable_args[i].recursive = 0; - handle->index_table[i] = MESA_htable_create(&indextable_args[i], sizeof(indextable_args[i])); - } - - return (GIE_handle_t *)(handle); -} - -int key_compare(const uchar * key1, uint size1, const uchar * key2, uint size2) -{ - return ( (*(long*)key1) - (*(long*)key2)); -} - - -void idtable_free(void * data) -{ - struct id_table_data * tmp = (struct id_table_data *)data; - free(tmp->sfh); - tmp->sfh = NULL; - tmp->tag = NULL; - free(tmp); - tmp = NULL; - - return; -} - -void indextable_delete_with_threshold(MESA_htable_handle * htable_handle, struct index_table_data * tmp, char * key) -{ - int key_length = strnlen(key,KEY_MAX_LENGTH); - struct linklist_node * tmp_node = TAILQ_FIRST(tmp->listhead); - while(tmp_node != NULL) - { - struct linklist_node * linklist_tmp = TAILQ_NEXT(tmp_node,listentry); - if(tmp_node->basicinfo->gram_cnt <= GRAM_CNT_THRESHOLD) - { - tmp_node = linklist_tmp; - continue; - } - TAILQ_REMOVE(tmp->listhead, tmp_node, listentry); - tmp_node->basicinfo->gram_cnt--; - tmp->cnt--; - if(TAILQ_EMPTY(tmp->listhead) == 1) - { - //_handle->hash_cnt--; - //_handle->mem_occupy -= (sizeof(struct index_table_data) + sizeof(struct TQ)); - if(MESA_htable_del(htable_handle, (const uchar *)(key), key_length, indextable_free) < 0) - { - printf("indextable backtrack delete error!\n"); - assert(0); - return; - } - } - //_handle->mem_occupy -= (sizeof(struct linklist_node) + sizeof(short)*(tmp_node->size)); - free(tmp_node->position); - tmp_node->position = NULL; - free(tmp_node); - tmp_node = NULL; - tmp_node = linklist_tmp; - - } - return; -} - - -void indextable_free(void * data) -{ - struct index_table_data * tmp = (struct index_table_data *)data; - struct linklist_node * tmp_node = TAILQ_FIRST(tmp->listhead); - while(tmp_node != NULL) - { - struct linklist_node * linklist_tmp = TAILQ_NEXT(tmp_node, listentry); - TAILQ_REMOVE(tmp->listhead, tmp_node, listentry); - tmp->cnt--; - free(tmp_node->position); - tmp_node->position = NULL; - free(tmp_node); - tmp_node = NULL; - tmp_node = linklist_tmp; - } - free(tmp->listhead); - tmp->listhead = NULL; - free(tmp); - tmp = NULL; - return; -} - - -void indextable_free_cnt(void * data) -{ - struct index_table_data * tmp = (struct index_table_data *)data; - hash_cnt++; - cnt_sum += tmp->cnt; - struct linklist_node * tmp_node = TAILQ_FIRST(tmp->listhead); - while(tmp_node != NULL) - { - struct linklist_node * linklist_tmp = TAILQ_NEXT(tmp_node, listentry); - TAILQ_REMOVE(tmp->listhead, tmp_node, listentry); - tmp->cnt--; - free(tmp_node->position); - tmp_node->position = NULL; - free(tmp_node); - tmp_node = NULL; - tmp_node = linklist_tmp; - } - free(tmp->listhead); - tmp->listhead = NULL; - free(tmp); - tmp = NULL; - return; -} - -void print_item_iterate_idtable(const uchar * key, uint size, void * data, void * user) -{ - struct id_table_data * id_data = (struct id_table_data *)data; - printf("id:%u\n",id_data->id); -} - - - -void print_item_iterate(const uchar * key, uint size, void * data, void * user) -{ - struct index_table_data * index_data = (struct index_table_data *)data; - printf("%s %d\n", (char *)key, index_data->cnt); - struct linklist_node * tmp_node = NULL; - int i = 0; - TAILQ_FOREACH(tmp_node, index_data->listhead, listentry) - { - printf("id = %u\n",tmp_node->basicinfo->id); - printf("position is :\n"); - for(i = 0;i < tmp_node->index;i++) - { - printf("%d ",tmp_node->position[i]); - } - printf("\n"); - } - printf("\n"); -} - -int edit_distn(const char *s1, int s1len, const char *s2, int s2len) -{ - long int max_len = 0; - if(s1len >= s2len) - { - max_len = s1len; - } - else - { - max_len = s2len; - } - int **t = (int **)malloc(2*sizeof(int *)); - t[0] = (int *)malloc((max_len +1)*sizeof(int)); - t[1] = (int *)malloc((max_len +1)*sizeof(int)); - //int t[2][EDIT_DISTN_MAXLEN+1]; - int *t1 = t[0]; - int *t2 = t[1]; - int *t3; - size_t i1, i2; - for (i2 = 0; i2 <= s2len; i2++) - t[0][i2] = i2 * EDIT_DISTN_REMOVE_COST; - for (i1 = 0; i1 < s1len; i1++) { - t2[0] = (i1 + 1) * EDIT_DISTN_INSERT_COST; - for (i2 = 0; i2 < s2len; i2++) { - int cost_a = t1[i2+1] + EDIT_DISTN_INSERT_COST; - int cost_d = t2[i2] + EDIT_DISTN_REMOVE_COST; - int cost_r = t1[i2] + (s1[i1] == s2[i2] ? 0 : EDIT_DISTN_REPLACE_COST); - t2[i2+1] = MIN(MIN(cost_a, cost_d), cost_r); - } - t3 = t1; - t1 = t2; - t2 = t3; - } - long int ret = t1[s2len]; - free(t[0]); - free(t[1]); - free(t); - return ret; - //return t1[s2len]; -} - - -void GIE_destory(GIE_handle_t * handle) -{ - GIE_handle_inner_t * _handle = (GIE_handle_inner_t *)(handle); - //printf("hash_cnt:%llu\n",_handle->hash_cnt); - //printf("mem_occupy:%llu\n",_handle->mem_occupy); - int i = 0; - for(i = 0;i < HTABLE_NUM;i++) - { - MESA_htable_destroy(_handle->index_table[i], indextable_free_cnt); - } - MESA_htable_destroy(_handle->id_table, idtable_free); - //printf("index_free hash_cnt :%llu\n", hash_cnt); - //printf("cnt sum :%llu\n",cnt_sum); - free(_handle); - _handle = NULL; -} - - -int grab_key_set(char * str_begin,short str_length,int i,unsigned int gram_value,short * gram_cnt,struct KL** to_process_list, unsigned long long blocksize) -{ - int k = 0,j = 0; - char * tmp_gram = str_begin; - char key[gram_value+1]; - int sum = 0,htable_index = 0; - if(str_length < gram_value) - { - return 0; - } - str_length = MIN(str_length,strnlen(str_begin,str_length)); - *gram_cnt = str_length - gram_value + 1; - //printf("str_length:%d\n",str_length); - for(k = 0; k < str_length - gram_value + 1; k++) - { - sum = 0; - memset(key,'\0', gram_value+1); - memcpy(key, tmp_gram++, gram_value); - //printf("k:%d key:%s\n",k,key); - for(j = 0; j < gram_value; j++) - { - sum += key[j]; - } - htable_index = sum%HTABLE_NUM; - struct key_list_node *tmp_node = (struct key_list_node *)calloc(1,sizeof(struct key_list_node)); - tmp_node->key = (char *)calloc(gram_value+1,sizeof(char)); - memcpy(tmp_node->key,key,gram_value); - tmp_node->digest_id = i; - tmp_node->pos = k; - tmp_node->blocksize = blocksize; - TAILQ_INSERT_TAIL(to_process_list[htable_index], tmp_node, keylistentry); - } - return 1; -} -int sfh_grab_key_set(char *sfh,short sfh_length,int i,unsigned int gram_value,short * gram_cnt,struct KL** to_process_list) -{ - int t = 0; - char * tmp_gram = sfh; - unsigned long long blocksize = 0; - for(t = 0; t < 2;t++) - { - blocksize = get_blocksize_from_head(tmp_gram, sfh_length); - while(*tmp_gram != '\0') - { - if(*tmp_gram == ':') - { - tmp_gram++; - break; - } - tmp_gram++; - } - unsigned int real_length = get_real_length(tmp_gram, sfh_length); - if(real_length < gram_value) - { - if(t==0) - { - return 0; - } - else - { - continue; - } - } - grab_key_set(tmp_gram, real_length, i, gram_value, gram_cnt, to_process_list, blocksize); - while(*tmp_gram != '\0') - { - if(*tmp_gram == '#') - { - tmp_gram++; - break; - } - tmp_gram++; - } - } - return 1; -} - -void free_key_set(struct KL ** to_process_list,int size) -{ - int i = 0; - for(i = 0;i < size;i++) - { - struct key_list_node *tmp_node = TAILQ_FIRST(to_process_list[i]); - while(tmp_node != NULL) - { - struct key_list_node *key_list_tmp = TAILQ_NEXT(tmp_node, keylistentry); - TAILQ_REMOVE(to_process_list[i], tmp_node, keylistentry); - free(tmp_node->key); - tmp_node->key = NULL; - free(tmp_node); - tmp_node = NULL; - tmp_node = key_list_tmp; - } - free(to_process_list[i]); - to_process_list[i]= NULL; - } -} - -int GIE_update(GIE_handle_t * handle,GIE_digest_t * * digests,int size) -{ - GIE_handle_inner_t * _handle = (GIE_handle_inner_t *)(handle); - struct id_table_data * info = NULL; - int success_cnt = 0; - int m = 0, i = 0, grab_ret = 0; - short gram_cnt = 0; - unsigned int input_fh_len = 0; - unsigned int gram_value = _handle->user_gram_value; - struct KL* to_process_list[HTABLE_NUM]; - - MESA_htable_handle htable_index_copy; - MESA_htable_handle htable_id_copy; - MESA_htable_handle htable_tmp_index=NULL,htable_tmp_id=NULL; - struct htable_handle * htable_copied_id_para = (struct htable_handle *)calloc(1,sizeof(struct htable_handle)); - struct htable_handle * htable_copied_index_para = (struct htable_handle *)calloc(1,sizeof(struct htable_handle)); - - htable_copied_id_para->runtime_table = _handle->id_table; - htable_copied_id_para->para = NULL; - htable_id_copy = copy_htable((void *)htable_copied_id_para, copy_idtable_item_iterate,idtable_free); - - MESA_htable_handle garbage_htable[HTABLE_NUM]; - /*if(MESA_htable_iterate(htable_id_copy, print_item_iterate_idtable, NULL) == -1) - { - printf("iterate error!\n"); - } - printf("size:%u\n",id_size);*/ - - for(m = 0;m < HTABLE_NUM;m++) - { - to_process_list[m]=(struct KL*)calloc(1,sizeof(struct KL)); - TAILQ_INIT(to_process_list[m]); - } - - for(i = 0; i < size; i++) - { - switch(digests[i]->operation) - { - case GIE_INSERT_OPT: - { - assert(digests[i]->tag!=NULL); - if(_handle->input_format == GIE_INPUT_FORMAT_SFH) - { - grab_ret = sfh_grab_key_set(digests[i]->sfh,digests[i]->sfh_length,i,gram_value,&gram_cnt,to_process_list); - } - else if(_handle->input_format == GIE_INPUT_FORMAT_PLAIN) - { - - grab_ret = grab_key_set(digests[i]->sfh,digests[i]->sfh_length,i,gram_value,&gram_cnt,to_process_list,0); - } - if(grab_ret == 0) - { - continue; - } - else - { - info = (struct id_table_data *)calloc(1,sizeof(struct id_table_data)); - input_fh_len = digests[i]->sfh_length; - info->sfh = (char *)calloc(input_fh_len + 1,sizeof(char)); - memcpy(info->sfh, digests[i]->sfh, input_fh_len); - _handle->mem_occupy += sizeof(struct id_table_data) + sizeof(char)*(input_fh_len+1); - info->sfh_length = digests[i]->sfh_length; - info->gram_cnt = gram_cnt; - - /*int tag_len = strnlen(digests[i]->tag,MAX_LENGTH); - info->tag = (char *)calloc(tag_len+1,sizeof(char)); - memcpy(info->tag,digests[i]->tag,tag_len);*/ - info->tag = digests[i]->tag; - - info->id = digests[i]->id; - info->cfds_lvl = digests[i]->cfds_lvl; - if(_handle->input_format == GIE_INPUT_FORMAT_SFH) - { - info->blocksize = get_blocksize_from_head(digests[i]->sfh, digests[i]->sfh_length); - } - else if(_handle->input_format == GIE_INPUT_FORMAT_PLAIN) - { - info->blocksize = 0; - } - - if(MESA_htable_add(htable_id_copy, (const uchar *)(&(digests[i]->id)), sizeof(digests[i]->id), (const void *)info) < 0) - { - _handle->mem_occupy -= (sizeof(struct id_table_data) + sizeof(char)*(input_fh_len+1)); - free(info->sfh); - info->sfh = NULL; - free(info); - info = NULL; - continue; - } - } - success_cnt ++; - break; - } - - case GIE_DELETE_OPT: - { - - struct id_table_data * ret = (struct id_table_data *) MESA_htable_search(htable_id_copy, \ - (const uchar *)(&(digests[i]->id)), sizeof(digests[i]->id)); - if(ret!= NULL) - { - if(_handle->input_format == GIE_INPUT_FORMAT_SFH) - { - success_cnt += sfh_grab_key_set(ret->sfh,ret->sfh_length,i,gram_value,&gram_cnt,to_process_list); - } - else if(_handle->input_format == GIE_INPUT_FORMAT_PLAIN) - { - - success_cnt += grab_key_set(ret->sfh,ret->sfh_length,i,gram_value,&gram_cnt,to_process_list,0); - } - } - else - { - break; - } - if(MESA_htable_del(htable_id_copy, (const uchar *)(&(digests[i]->id)), sizeof(digests[i]->id), idtable_free) < 0) - { - printf("delete id failed!"); - assert(0); - } - //success_cnt += GIE_delete(_handle, digests[i]); - break; - } - - default: - break; - } - - } - unsigned int digest_id = 0; - struct id_table_data * tmp_info= NULL; - - for(i = 0;i < HTABLE_NUM;i++) - { - htable_copied_index_para->runtime_table = _handle->index_table[i]; - htable_copied_index_para->para = htable_id_copy; - htable_index_copy = copy_htable((void *)htable_copied_index_para,copy_indextable_item_iterate,indextable_free); - struct key_list_node * tmp_node; - TAILQ_FOREACH(tmp_node, to_process_list[i], keylistentry) - { - digest_id = tmp_node->digest_id; - if(digests[digest_id]->operation == GIE_INSERT_OPT) - { - tmp_info =(struct id_table_data *)MESA_htable_search(htable_id_copy, (const uchar *)(&(digests[digest_id])->id), \ - sizeof((digests[digest_id])->id)); - if(tmp_info == NULL) - { - printf("id %u not insert\n",digests[digest_id]->id); - } - if(GIE_insert_indextable(htable_index_copy, tmp_info, tmp_node->key, tmp_node->pos,tmp_node->blocksize) < 0) - { - printf("insert %d indextable failed!\n",digests[digest_id]->id); - continue; - } - } - else if(digests[digest_id]->operation == GIE_DELETE_OPT) - { - if(GIE_delete_from_indextable_by_key(htable_index_copy, tmp_node->key, (digests[digest_id])->id) < 0) - { - printf("delete %d indextable failed!\n",digests[digest_id]->id); - continue; - } - } - } - htable_tmp_index= _handle->index_table[i]; - _handle->index_table[i] = htable_index_copy; - garbage_htable[i]=htable_tmp_index; - } - - htable_tmp_id = _handle->id_table; - _handle->id_table = htable_id_copy; - usleep(200); - MESA_htable_destroy(htable_tmp_id, idtable_free); - /*if(MESA_htable_iterate(_handle->index_table, print_item_iterate, NULL) == -1) - { - printf("iterate error!\n"); - }*/ - for(i=0;iruntime_table = copy_htable_handle; - htable_iterate_para->para = htable_copied_para->para; - - if(MESA_htable_iterate(htable_copied_para->runtime_table, func, htable_iterate_para) == -1) - { - printf("iterate error!\n"); - } - free(htable_iterate_para); - htable_copied_para=NULL; - return copy_htable_handle; -} - -void copy_indextable_item_iterate(const uchar * key, uint size, void * data, void * user) -{ - struct index_table_data * index_data = (struct index_table_data *)data; - struct htable_handle * htable_copied_para = (struct htable_handle *)user; - - struct index_table_data * index_data_copy = (struct index_table_data *)calloc(1, sizeof(struct index_table_data)); - struct TQ * head = (struct TQ *)calloc(1, sizeof(struct TQ)); - index_data_copy->listhead = head; - index_data_copy->cnt = index_data->cnt; - - TAILQ_INIT(head); - struct linklist_node * tmp_node = NULL; - struct id_table_data * ret = NULL; - int i = 0; - - TAILQ_FOREACH(tmp_node, index_data->listhead, listentry) - { - struct linklist_node * node_data = (struct linklist_node *)calloc(1,sizeof(struct linklist_node)); - node_data->size = tmp_node->size; - node_data->position = (short *)calloc(node_data->size, sizeof(short)); - for(i = 0;i < tmp_node->index;i++) - { - node_data->position[i] = tmp_node->position[i]; - } - ret = (struct id_table_data *)MESA_htable_search(htable_copied_para->para, (const uchar *)(&(tmp_node->basicinfo->id)), sizeof(tmp_node->basicinfo->id)); - if(ret == NULL) - { - //printf("copy id %u not exist\n",tmp_node->basicinfo->id); - free(node_data->position); - node_data->position = NULL; - free(node_data); - node_data = NULL; - continue; - } - node_data->basicinfo = ret; - node_data->index = tmp_node->index; - node_data->blocksize = tmp_node->blocksize; - TAILQ_INSERT_TAIL(head, node_data, listentry); - } - MESA_htable_add(htable_copied_para->runtime_table, key, size, (const void *)index_data_copy); -} -//TODO: Using the orginal value instead of make a duplication to be faster. -void copy_idtable_item_iterate(const uchar * key, uint size, void * data, void * user) -{ - struct id_table_data * id_data = (struct id_table_data *)data; - struct htable_handle * htable_para = (struct htable_handle *)user; - struct id_table_data * id_data_copy = (struct id_table_data *)calloc(1, sizeof(struct id_table_data)); - assert(id_data->tag!=NULL); - memcpy(id_data_copy,id_data,sizeof(struct id_table_data)); - id_data_copy->sfh = (char *)calloc(id_data_copy->sfh_length,sizeof(char)); - memcpy(id_data_copy->sfh,id_data->sfh,id_data_copy->sfh_length); - - MESA_htable_add(htable_para->runtime_table, (const uchar *)(&(id_data_copy->id)), sizeof(id_data_copy->id), (const void *)id_data_copy); -} - - - - -int GIE_insert_indextable(MESA_htable_handle htable_copy, struct id_table_data * info, char * key, unsigned int index, unsigned long long blocksize) -{ - int key_length = strnlen(key,KEY_MAX_LENGTH); - struct linklist_node * node_data = (struct linklist_node *)calloc(1,sizeof(struct linklist_node)); - node_data->size = GRAM_CNT_MAX; - node_data->position = (short *)calloc(node_data->size, sizeof(short)); - node_data->basicinfo = info; - node_data->index = 0; - node_data->position[(node_data->index)++] = index; - node_data->blocksize = blocksize; - - //_handle->mem_occupy += sizeof(struct linklist_node) + sizeof(short)*(node_data->size); - - struct index_table_data * ret = (struct index_table_data *)(MESA_htable_search(htable_copy, \ - (const uchar *)(key), key_length)); - - - if(ret != NULL) - { - struct linklist_node * tmp = NULL; - TAILQ_FOREACH(tmp, ret->listhead, listentry) - { - if(tmp->basicinfo->id > node_data->basicinfo->id) - { - TAILQ_INSERT_BEFORE(tmp, node_data, listentry); - ret->cnt ++; - if(ret->cnt >= CNT_MAX) - { - indextable_delete_with_threshold(htable_copy,ret,key); - } - return 0; - } - if(tmp->basicinfo->id == node_data->basicinfo->id && tmp->blocksize == blocksize) - { - if(tmp->index >= tmp->size) - { - tmp->size *= 2; - tmp->position = realloc(tmp->position, (tmp->size)*sizeof(short)); - } - tmp->position[(tmp->index)++] = index; - //_handle->mem_occupy -= (sizeof(struct linklist_node) + sizeof(short)*(node_data->size)); - free(node_data->position); - node_data->position = NULL; - free(node_data); - node_data = NULL; - return 0; - } - } - TAILQ_INSERT_TAIL(ret->listhead, node_data, listentry); - ret->cnt ++; - if(ret->cnt >= CNT_MAX) - { - indextable_delete_with_threshold(htable_copy,ret,key); - } - } - - else - { - struct index_table_data * index_data = (struct index_table_data *)calloc(1, sizeof(struct index_table_data)); - struct TQ * head = (struct TQ *)calloc(1, sizeof(struct TQ)); - //_handle->mem_occupy += sizeof(struct index_table_data) + sizeof(struct TQ); - - index_data->listhead = head; - index_data->cnt = 0; - - TAILQ_INIT(head); - TAILQ_INSERT_TAIL(head, node_data, listentry); - index_data->cnt++; - //_handle->hash_cnt++; - if(MESA_htable_add(htable_copy, (const uchar *)(key), key_length, (const void *)index_data) < 0) - { - printf("add index_table failed!\n"); - assert(0); - return -1; - } - } - return 0; - -} - - - -int GIE_delete(GIE_handle_inner_t * _handle, GIE_digest_t * digest) -{ - int success_cnt = 0; - struct id_table_data * ret = (struct id_table_data *) MESA_htable_search(_handle->id_table, \ - (const uchar *)(&(digest->id)), sizeof(digest->id)); - if(ret == NULL) - { - printf("del %d doesn't exist!\n",digest->id); - return -1; - } - else - { - int gram_value = _handle->user_gram_value; - char key[gram_value+1]; - char * tmp_gram = ret->sfh; - while(*tmp_gram != '\0') - { - if(*tmp_gram == ':') - { - tmp_gram++; - break; - } - tmp_gram++; - } - unsigned int real_length = get_real_length(tmp_gram, ret->sfh_length); - int gram_cnt = real_length - gram_value + 1; - int k = 0; - for(k = 0; k < gram_cnt; k++) - { - memset(key, '\0', gram_value+1); - memcpy(key, tmp_gram++, gram_value); - if(GIE_delete_from_indextable_by_key(_handle, key, digest->id) < 0) - { - printf("delete %d indextable failed!\n",digest->id); - continue; - } - } - success_cnt++; - } - - return success_cnt; -} - - - -int GIE_delete_from_indextable_by_key(MESA_htable_handle htable, char * key, unsigned int id) -{ - int key_length = strnlen(key,KEY_MAX_LENGTH); - struct index_table_data * ret = (struct index_table_data *)(MESA_htable_search(htable, \ - (const uchar *)(key), key_length)); - if(ret == NULL) - { - return 0; - } - - - struct linklist_node * tmp = TAILQ_FIRST(ret->listhead); - while(tmp != NULL) - { - struct linklist_node * linklist_tmp = TAILQ_NEXT(tmp, listentry); - if(tmp->basicinfo->id != id) - { - tmp=linklist_tmp; - continue; - } - TAILQ_REMOVE(ret->listhead, tmp, listentry); - ret->cnt--; - //_handle->mem_occupy -= (sizeof(struct linklist_node) + sizeof(short)*(tmp->size)); - free(tmp->position); - tmp->position = NULL; - free(tmp); - tmp = NULL; - if(TAILQ_EMPTY(ret->listhead) == 1) - { - //_handle->mem_occupy -= (sizeof(struct index_table_data) + sizeof(struct TQ)); - int ret = MESA_htable_del(htable, (const uchar *)(key), key_length, indextable_free); - if(ret < 0) - { - printf("indextable backtrack delete error!\n"); - assert(0); - return -1; - } - - } - } - return 0; -} - - - - -int GIE_cmp(const void * a, const void * b) -{ - unsigned int tmp_a = *(unsigned int *)a; - unsigned int tmp_b = *(unsigned int *)b; - if(before(tmp_a, tmp_b)) - { - return -1; - } - else if(after(tmp_a, tmp_b)) - { - return 1; - } - else - { - return 0; - } -} - - -static inline unsigned int get_real_length(const char * string, unsigned int length) -{ - unsigned int ret = 0; - const char * tmp_str = string; - while(*tmp_str != '\0') - { - if(*tmp_str == '[') - { - break; - } - tmp_str++; - ret ++; - } - return ret; -} - - -static inline int GIE_part_query(GIE_handle_inner_t * _handle, const char * query_string, int index_begin, int part_query_len,unsigned int ** id_union, unsigned int * union_index, unsigned int * union_size, unsigned long long blocksize) -{ - unsigned int gram_value = _handle->user_gram_value; - - unsigned int real_length = part_query_len; - unsigned int chunk_count_max = 0; - if(real_length < gram_value) - { - return 0; - } - else - { - chunk_count_max = real_length/gram_value; - } - char key[gram_value+1]; - struct index_table_data * ret = NULL; - struct linklist_node * tmp_node_t = NULL; - - unsigned int position_accuracy = _handle->user_position_accuracy; - - int i=0,j=0,k=0; - unsigned int tmp_min = 0; - int sum = 0, htable_index = 0; - for(i = index_begin; i < chunk_count_max + index_begin; i++) - { - sum = 0; - memset(key,'\0',gram_value+1); - memcpy(key, query_string, gram_value); - for(k = 0; k < gram_value; k++) - { - sum += key[k]; - } - htable_index = sum%HTABLE_NUM; - ret = (struct index_table_data *) MESA_htable_search(_handle->index_table[htable_index], \ - (const uchar *)(key), strnlen(key,gram_value)); - query_string = query_string + gram_value; - - if(ret ==NULL) - { - break; - } - - tmp_node_t = NULL; - TAILQ_FOREACH(tmp_node_t, ret->listhead, listentry) - { - tmp_min = 0; - if(i*gram_value >= position_accuracy) - { - tmp_min = i*gram_value - position_accuracy; - } - for(j = 0; j < tmp_node_t->index; j++) - { - if((blocksize == tmp_node_t->basicinfo->blocksize) && (tmp_node_t->position[j] >= tmp_min) && (tmp_node_t->position[j] <= i*gram_value + position_accuracy)) - //if(blocksize == tmp_node_t->basicinfo->blocksize) - { - if((*union_index) >= (*union_size)) - { - *union_size = (*union_size) * 2; - *id_union = (unsigned int *)realloc(*id_union, (*union_size)*sizeof(unsigned int)); - } - (*id_union)[(*union_index)] = tmp_node_t->basicinfo->id; - (*union_index)++; - break; - } - } - } - } - return chunk_count_max; -} - -static inline int GIE_gram_with_position(GIE_handle_inner_t * _handle, unsigned long long query_blocksize, const char * fuzzy_string, unsigned int ** id_union, - unsigned int * union_index,unsigned int * union_size, unsigned int * chunk_cnt) -{ - const char * tmpstr = fuzzy_string; - const char * query_string_begin; - unsigned long long blocksize = query_blocksize; - int part_query_len = 0; - int query_actual_len = 0; - while(*tmpstr != ':'&& *tmpstr != '\0') - { - tmpstr ++; - } - if(*tmpstr == ':') - { - tmpstr ++; - } - else - { - return 0; - } - query_string_begin = tmpstr; - char *p = NULL; - - while((*query_string_begin) != '\0') - { - int left = 0; - int right = 0; - p=strchr(query_string_begin,'['); - if(p!=NULL) - { - part_query_len = p-query_string_begin; - int ret = sscanf(p,"[%d:%d]",&left,&right); - if(ret != 2) - { - break; - } - p=strchr(p,']'); - if(p != NULL && (*p) != '\0') - { - int index_begin = (left/blocksize - TOLERENCE_SIZE > 0 ? (left/blocksize - TOLERENCE_SIZE) : 0); - (*chunk_cnt) += GIE_part_query(_handle,query_string_begin,index_begin, part_query_len, - id_union, union_index, union_size, blocksize); - query_actual_len += part_query_len; - query_string_begin = p+1; - } - else - { - break; - } - } - else - { - break; - } - } - return query_actual_len; -} - -inline unsigned long long calc_fh_blocksize(unsigned long long orilen) -{ - double tmp = orilen/(64 * BLOCKSIZE_MIN); - double index = floor(log(tmp)/log(2)); - double tmp_t = pow(2,index); - unsigned long long blocksize = (unsigned long long)(tmp_t * BLOCKSIZE_MIN); - return blocksize; -} - -static inline unsigned long long get_blocksize_from_head(const char * fuzzy_string, unsigned int str_len) -{ - const char * tmp_str = fuzzy_string; - char blk[100]; - memset(blk,'\0',sizeof(blk)); - unsigned long long blocksize = 0; - int i = 0; - while(*tmp_str != '\0' && *tmp_str != ':' && str_len != 0 && i < 100) - { - blk[i++] = *tmp_str; - tmp_str++; - str_len--; - } - blocksize = (unsigned long long)atoi(blk); - return blocksize; -} -int GIE_string_similiarity(const char *str1, int len1, const char *str2, int len2) -{ - int edit_distance=0; - int conf=0; - edit_distance = edit_distn(str1, len1,str2,len2); - conf = 100-(edit_distance*100)/(len1 + len2); - return conf; -} - -int GIE_sfh_similiarity(const char *sfh1, int len1, const char *sfh2, int len2) -{ - int j = 0, t = 0; - unsigned long long query_blocksize = 0, index_blocksize = 0; - unsigned int query_real_length = 0, index_real_length = 0; - const char *query_gram_begin = sfh1; - const char *index_gram_begin = sfh2; - char *splice_str = (char *)malloc(sizeof(char)*len1); - memset(splice_str,'\0',len1); - char *spli_str_begin = splice_str; - int edit_distance = 0; - int ret = 0; - char *p = NULL; - int splice_len = 0; - - for(j = 0; j < 2; j++) - { - index_blocksize = get_blocksize_from_head(index_gram_begin, len2); - while((*index_gram_begin) != '\0') - { - if((*index_gram_begin) == ':') - { - index_gram_begin++; - break; - } - index_gram_begin++; - } - index_real_length = get_real_length(index_gram_begin, len2); - query_gram_begin = sfh1; - for(t = 0; t < 2; t++) - { - query_blocksize = get_blocksize_from_head(query_gram_begin, len1); - //printf("gram_begin:%c\n",*index_gram_begin); - //printf("gram_str:%s\n",index_gram_begin); - while((*query_gram_begin) != '\0') - { - if((*query_gram_begin) == ':') - { - query_gram_begin++; - break; - } - query_gram_begin++; - } - //printf("query_blocksize:%lld, index_blocksize:%lld\n",query_blocksize,index_blocksize); - //index_real_length = get_real_length(index_gram_begin, len1); - if(query_blocksize == index_blocksize) - { - while((*query_gram_begin) != '#' && (*query_gram_begin) != '\0') - { - p=strchr(query_gram_begin,'['); - if(p!=NULL) - { - query_real_length = p-query_gram_begin; - p=strchr(p,']'); - if(p != NULL && (*p) != '\0') - { - - memcpy(spli_str_begin,query_gram_begin,query_real_length); - spli_str_begin += query_real_length; - //edit_distance += edit_distn(query_gram_begin, query_real_length, index_gram_begin, index_real_length); - query_gram_begin = p+1; - } - else - { - break; - } - } - else - { - break; - } - } - splice_len = strnlen(splice_str,len1); - edit_distance = edit_distn(index_gram_begin, index_real_length, splice_str, splice_len); - //printf("query_real_length:%d splice_length:%d edit_distance:%d\n",query_real_length,splice_len,edit_distance); - ret = 100-(edit_distance*100)/(index_real_length + splice_len); - //ret = (100*ret)/SPAM_LENGTH; - //ret = 100-ret; - //ret = 100 - (100*edit_distance)/(query_real_length); - free(splice_str); - return ret; - } - while(*query_gram_begin != '\0') - { - if(*query_gram_begin == '#') - { - query_gram_begin++; - break; - } - query_gram_begin++; - } - - } - while(*index_gram_begin != '\0') - { - if(*index_gram_begin == '#') - { - index_gram_begin++; - break; - } - index_gram_begin++; - } - } - //printf("no blocksize:query_real_length:%d splice_length:%d edit_distance:%d\n",query_real_length,splice_len,edit_distance); - free(splice_str); - return 0; -} - - - - -int GIE_query(GIE_handle_t * handle, const char * data, int data_len, GIE_result_t * results, int result_size) -{ - GIE_handle_inner_t * _handle = (GIE_handle_inner_t *) handle; - int i = 0, j = 0; - unsigned int union_index = 0; - unsigned int gram_value = _handle->user_gram_value; - unsigned int query_actual_len = 0; - unsigned int union_size = UNION_INIT_SIZE; - unsigned int chunk_cnt = 0; - const char *fuzzy_string_begin = data; - unsigned int * id_union =(unsigned int *)calloc(union_size, sizeof(unsigned int)); - unsigned long long query_blocksize = 0; - unsigned int fuzzy_string_len = (unsigned int)data_len; - - if(_handle->input_format == GIE_INPUT_FORMAT_SFH) - { - for(j = 0;j < 2;j++) - { - query_blocksize = get_blocksize_from_head(fuzzy_string_begin, fuzzy_string_len); - if(query_blocksize == 0) - { - return 0; - } - query_actual_len += GIE_gram_with_position(_handle, query_blocksize, fuzzy_string_begin, &id_union, &union_index, &union_size, &chunk_cnt); - while(*fuzzy_string_begin != '#' && *fuzzy_string_begin != '\0') - { - fuzzy_string_begin++; - } - if(*fuzzy_string_begin == '#') - { - fuzzy_string_begin++; - } - } - } - else if(_handle->input_format == GIE_INPUT_FORMAT_PLAIN) - { - query_actual_len = fuzzy_string_len; - chunk_cnt = GIE_part_query(_handle, fuzzy_string_begin, 0, query_actual_len, &id_union, &union_index, &union_size, 0); - } - - if(union_index == 0) - { - free(id_union); - id_union = NULL; - return 0; - } - - qsort(id_union, union_index, sizeof(id_union[0]), GIE_cmp); - - unsigned int current_id = id_union[0]; - unsigned int * tmp_id = id_union; - unsigned int count = 0; - struct id_table_data * ret_tmp = NULL; - short conf = 0; - int ret_size = 0; - for(i = 0; i <= union_index; i++) - { - if( i == union_index || *tmp_id != current_id ) - { - ret_tmp = (struct id_table_data *) MESA_htable_search(_handle->id_table, \ - (const uchar *)(&(current_id)), sizeof(current_id)); - - if(ret_tmp == NULL) - { - break; - } - char * tmp_gram = ret_tmp->sfh; - int length = ret_tmp->sfh_length; - if(ret_tmp->gram_cnt == 0||chunk_cnt == 0) - { - conf = 0; - } - else - { - conf = (count*(query_actual_len-gram_value+1)*10)/(chunk_cnt*(ret_tmp->gram_cnt)); - } - - if(_handle->ED_reexamine == 1) - { - if(_handle->input_format == GIE_INPUT_FORMAT_SFH) - { - conf = GIE_sfh_similiarity(data, fuzzy_string_len, tmp_gram, length); - } - else - { - conf=GIE_string_similiarity(data, fuzzy_string_len, tmp_gram, length); - } - } - - if(conf >= ret_tmp->cfds_lvl) - { - results[ret_size].cfds_lvl = conf; - results[ret_size].id = current_id; - /*results[ret_size].tag = (char *)malloc((ret_tmp->sfh_length + 1)*sizeof(char)); - memset(results[ret_size].tag,'\0',(ret_tmp->sfh_length+1)); - memcpy(results[ret_size].tag, ret_tmp->sfh,ret_tmp->sfh_length);*/ - results[ret_size].tag = ret_tmp->tag; - ret_size++; - } - - if(ret_size == result_size) - { - break; - } - - current_id = *tmp_id; - count = 1; - - } - else - { - count++; - } - - tmp_id ++; - } - - free(id_union); - id_union = NULL; - return ret_size; -} - - -unsigned long long GIE_status(GIE_handle_t * handle, int type) -{ - unsigned long long length; - GIE_handle_inner_t * _handle = (GIE_handle_inner_t *)handle; - switch(type) - { - case MEM_OCCUPY: - length = _handle->mem_occupy; - break; - default: - return 0; - } - return length; -} - diff --git a/src/entry/hyperscan_adapter.cpp b/src/entry/hyperscan_adapter.cpp deleted file mode 100644 index 8fc03fc..0000000 --- a/src/entry/hyperscan_adapter.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -struct hs_adapter -{ - hs_database_t *hs_pure_literal_db; - hs_database_t *hs_regex_db; - struct bool_matcher *logical_matcher; - int mode; - int n_thread; - size_t regex_cnt; - size_t regex_sub_pattern_cnt; - size_t pure_lit_cnt; - size_t pure_lit_sub_pattern_cnt; - void **user_tag_array; - hs_scratch_t *scratchs[]; -}; -struct hs_adapter *hs_adapter_new(struct hs_expression ** exprs, size_t n_expr, int scan_mode, int n_thread) -{ - struct hs_adapter *hsa=ALLOC(struct hs_adapter, 1); - size_t i=0, j=0; - unsigned int *hs_ids=NULL; - for(i=0; iregex_cnt++; - hsa->regex_sub_pattern_cnt=exprs[i].n_sub_pattern; - } - else - { - hsa->pure_lit_cnt++; - hsa->pure_lit_sub_pattern_cnt++; - } - } - const char **regex_exprs=NULL, **pure_lit_exprs=NULL; - if(hsa->regex_sub_pattern_cnt>0) - { - regex_exprs=ALLOC(char *, hsa->regex_sub_pattern_cnt); - } - if(hsa->pure_lit_sub_pattern_cnt>0) - { - pure_lit_exprs=ALLOC(char *, hsa->pure_lit_sub_pattern_cnt) - } - size_t sub_pure_lit_idx=0, sub_regex_idx=0; - for(i=0; in_sub_pattern; j++) - { - regex_exprs[j]=exprs[i]. - } - sub_pure_lit_idx - - } - struct bool_expr *bool_exprs=ALLOC(struct bool_expr, 1); -} -void hs_adpter_free(struct hs_adapter *adapter) -{ - -} -int hs_adapter_scan(struct hs_adapter *adapter, int thread_id, const char* data, unsigned int length, void **matched_tags, size_t n_tag) -{ -} -int hs_adapter_open_stream(struct hs_adapter *adapter, int thread_id) -{ - -} \ No newline at end of file diff --git a/src/entry/interval_index.c b/src/entry/interval_index.c deleted file mode 100644 index b993a3f..0000000 --- a/src/entry/interval_index.c +++ /dev/null @@ -1,737 +0,0 @@ -/********************************************************************* - * File: - * interval_index.c - * Author: - * TangQi - * E-mail: - * tangqi@iie.ac.cn - *********************************************************************/ -#include -#include -#include -#include "interval_index.h" -#include "rbtree.h" -#include "rbtree_augmented.h" - -/** - * There is a trick here. In order to hide specific - * realization of some structures, we use some approaches. - * Then the inner structure is named with "shadow", and - * the outer structure is named with "light". These words - * come from movie <>. Enjoy it :) - **/ - - -/** - * Structure of inner segment - **/ -typedef struct __IVI_shadow_seg_t{ - IVI_seg_t lightseg; /* interval for user, including left edge, right edge, and user's data */ - struct rb_node rb; /* node of rb-tree */ - OFFSET_TYPE max; /* max edge of subtree */ -}IVI_shadow_seg_t; - - -/* Structure of inner InterVal Index */ -typedef struct __IVI_shadow_t{ - struct rb_root root; - - /* statistics */ - int segs_cnt; - OFFSET_TYPE segs_length; - unsigned long long mem_occupy; //do not include user data -}IVI_shadow_t; - - - - - -IVI_seg_t * IVI_first_seg(IVI_t * handler) -{ - assert(handler != NULL); - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_node *first_node = rb_first(&(shadow_ivi->root)); - if(first_node == NULL) - return NULL; - return (IVI_seg_t *)(rb_entry(first_node, IVI_shadow_seg_t, rb)); -} - - -IVI_seg_t * IVI_last_seg(IVI_t * handler) -{ - assert(handler != NULL); - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_node *last_node = rb_last(&(shadow_ivi->root)); - if(last_node == NULL) - return NULL; - return (IVI_seg_t *)(rb_entry(last_node, IVI_shadow_seg_t, rb)); -} - - - -IVI_seg_t * IVI_prev_seg(IVI_seg_t * seg) -{ - assert(seg != NULL); - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg; - struct rb_node * prev_node = rb_prev(&(shadow_seg->rb)); - if(prev_node == NULL) - return NULL; - return (IVI_seg_t *)(rb_entry(prev_node, IVI_shadow_seg_t, rb)); -} - - - -IVI_seg_t * IVI_next_seg(IVI_seg_t * seg) -{ - assert(seg != NULL); - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg; - struct rb_node * next_node = rb_next(&(shadow_seg->rb)); - if(next_node == NULL) - return NULL; - return (IVI_seg_t *)(rb_entry(next_node, IVI_shadow_seg_t, rb)); -} - - -IVI_seg_t * IVI_prev_continuous_seg(IVI_seg_t * seg) -{ - assert(seg != NULL); - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg; - struct rb_node * prev_node = rb_prev(&(shadow_seg->rb)); - if(prev_node == NULL) - { - return NULL; - } - IVI_seg_t * prev_seg = (IVI_seg_t *)(rb_entry(prev_node, IVI_shadow_seg_t, rb)); - if(!continuous(prev_seg->right, seg->left)) - { - return NULL; - } - return prev_seg; -} - - -IVI_seg_t * IVI_next_continuous_seg(IVI_seg_t * seg) -{ - assert(seg != NULL); - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg; - struct rb_node * next_node = rb_next(&(shadow_seg->rb)); - if(next_node == NULL) - { - return NULL; - } - IVI_seg_t * next_seg = (IVI_seg_t *)(rb_entry(next_node, IVI_shadow_seg_t, rb)); - if(!continuous(seg->right, next_seg->left)) - { - return NULL; - } - return next_seg; -} - - -static inline int __is_overlapped(OFFSET_TYPE left1, OFFSET_TYPE right1, OFFSET_TYPE left2, OFFSET_TYPE right2) -{ - if(!after(left1, right2) && !after(left2, right1)) - return 1; - return 0; -} - - -/** - * Name: - * IVI_relative_position - * Description: - * Get relative position of given two interval segments - * Params: - * seg1: Subject of relation - * seg2: Object of relation - * Relation: - * On success, return the relation of two segments with enum; - * Else, return ERROR in enum; - **/ -Relation_t IVI_relative_position(IVI_seg_t * seg1, IVI_seg_t * seg2) -{ - if(NULL == seg1 || NULL == seg2) - { - return ERROR; - } - - if(before(seg1->right, seg2->left)) - { - return LEFT_NO_OVERLAP; - } - - if(!before(seg1->right, seg2->left) && before(seg1->right, seg2->right) && before(seg1->left, seg2->left)) - { - return LEFT_OVERLAP; - } - - if(!before(seg1->left, seg2->left) && !after(seg1->right, seg2->right)) - { - return CONTAINED; - } - - if(!after(seg1->left, seg2->left) && !before(seg1->right, seg2->right)) - { - return CONTAIN; - } - - if(!after(seg1->left, seg2->right) && after(seg1->right, seg2->right) && after(seg1->left, seg2->left)) - { - return RIGHT_OVERLAP; - } - - if(after(seg1->left, seg2->right)) - { - return RIGHT_NO_OVERLAP; - } - return ERROR; -} - - - -/** - * Name: - * IVI_create - * Description: - * Create an InterVal Index - * Params: - * void - * Return: - * Return a handler of this InterVal Index - **/ -IVI_t * IVI_create(void) -{ - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)malloc(sizeof(IVI_shadow_t)); - shadow_ivi->root = RB_ROOT; //init rb tree's root - shadow_ivi->segs_cnt = 0; - shadow_ivi->segs_length = 0; - shadow_ivi->mem_occupy = sizeof(IVI_shadow_t); - return (IVI_t *)shadow_ivi; -} - - - -static void __free_rb_tree(struct rb_node * root, IVI_callback_t cb, void * usr_para) -{ - if(root == NULL) - { - return; - } - if(root->rb_left != NULL) - { - __free_rb_tree(root->rb_left, cb, usr_para); - } - if(root->rb_right != NULL) - { - __free_rb_tree(root->rb_right, cb, usr_para); - } - /* free user data */ - IVI_shadow_seg_t * shadow_seg = rb_entry(root, IVI_shadow_seg_t, rb); - if(cb != NULL) - { - cb((IVI_seg_t *)shadow_seg, usr_para); - } - - /* free seg */ - free(shadow_seg); - shadow_seg = NULL; - return; -} - -/** - * Name: - * IVI_destroy - * Description: - * Destroy a given InterVal Index's handler - * Params: - * handler: The InterVal Index you want to destroy - * cb: Callback function for user to free data in segement - * usr_para: User parameter - * Return: - * void - **/ -void IVI_destroy(IVI_t * handler, IVI_callback_t cb, void * usr_para) -{ - if(handler == NULL) - { - return; - } - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - __free_rb_tree(shadow_ivi->root.rb_node, cb, usr_para); - free(shadow_ivi); - handler = NULL; - return; -} - - - -/** - * Name: - * IVI_seg_malloc - * Description: - * Malloc a segment with given parameters - * Params: - * left: Left point of segment - * right: Right point of segment - * data: User data - * Return: - * Return a pointer of segment structure. - **/ -IVI_seg_t * IVI_seg_malloc(OFFSET_TYPE left, OFFSET_TYPE right, void * data) -{ - /* Left must <= Right */ - if(after(left, right)) - { - return NULL; - } - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)malloc(sizeof(IVI_shadow_seg_t)); - shadow_seg->lightseg.left = left; - shadow_seg->lightseg.right= right; - shadow_seg->lightseg.data = data; - shadow_seg->max = 0; - - return (IVI_seg_t *)shadow_seg; -} - - - -/** - * Name: - * IVI_seg_free - * Description: - * Free the memory of given segment - * Params: - * seg: The segment that you want to free - * cb: Callback function for user to free *data in seg - * usr_para: User parameter for cb - * Return: - * void - **/ -void IVI_seg_free(IVI_seg_t * seg, IVI_callback_t cb, void * usr_para) -{ - assert(seg != NULL); - - /* Free user data first */ - if(cb != NULL) - { - cb(seg, usr_para); - } - IVI_shadow_seg_t * shadow_seg = (IVI_shadow_seg_t *)seg; - - /* Free seg */ - free(shadow_seg); - seg = NULL; -} - - - - -static inline OFFSET_TYPE __interval_tree_get_subtree_max(IVI_shadow_seg_t * node) -{ - OFFSET_TYPE max = node->lightseg.right, subtree_max; - if(node->rb.rb_left) - { - subtree_max = (rb_entry(node->rb.rb_left, IVI_shadow_seg_t, rb))->max; - if(before(max, subtree_max)) - max = subtree_max; - } - if(node->rb.rb_right) - { - subtree_max = (rb_entry(node->rb.rb_right, IVI_shadow_seg_t, rb))->max; - if(before(max, subtree_max)) - max = subtree_max; - } - return max; -} - - -static void __interval_tree_augment_propagate(struct rb_node * rb, struct rb_node * stop) -{ - while(rb != stop) - { - IVI_shadow_seg_t * node = rb_entry(rb, IVI_shadow_seg_t, rb); - OFFSET_TYPE subtree_max = __interval_tree_get_subtree_max(node); - if(node->max == subtree_max) - { - break; - } - node->max = subtree_max; - rb = rb_parent(&node->rb); - } - return; -} - - -static void __interval_tree_augment_copy(struct rb_node * rb_old, struct rb_node * rb_new) -{ - IVI_shadow_seg_t * old = rb_entry(rb_old, IVI_shadow_seg_t, rb); - IVI_shadow_seg_t * new = rb_entry(rb_new, IVI_shadow_seg_t, rb); - new->max = old->max; - return; -} - - -static void __interval_tree_augment_rotate(struct rb_node * rb_old, struct rb_node * rb_new) -{ - IVI_shadow_seg_t * old = rb_entry(rb_old, IVI_shadow_seg_t, rb); - IVI_shadow_seg_t * new = rb_entry(rb_new, IVI_shadow_seg_t, rb); - new->max = old->max; - old->max = __interval_tree_get_subtree_max(old); - return; -} - - -static const struct rb_augment_callbacks __interval_tree_augment_callbacks = { - __interval_tree_augment_propagate, - __interval_tree_augment_copy, - __interval_tree_augment_rotate -}; - - -/** - * Name: - * IVI_insert - * Description: - * Insert a segment to an InterVal Index handler,and the segment - * MUST not be overlapped with others in handler. - * Params: - * handler: The handler of InterVal Index created by IVI_create - * seg: A segment that user wants to add. It MUST be created - * by IVI_seg_malloc. - * Return: - * On success, 0 is returned; - * Else when overlapp occures or error occures, -1 is returned. - **/ -int IVI_insert(IVI_t * handler, IVI_seg_t * seg) -{ - if(NULL == handler || NULL == seg) - { - return -1; - } - - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_root * root = &(shadow_ivi->root); - OFFSET_TYPE left = seg->left, right = seg->right; - struct rb_node **link = &root->rb_node, *rb_parent = NULL; - IVI_shadow_seg_t * parent = NULL; - IVI_shadow_seg_t * new_seg = (IVI_shadow_seg_t *)seg; - while(*link) - { - rb_parent = *link; - parent = rb_entry(rb_parent, IVI_shadow_seg_t, rb); - /* is overlapped */ - if(__is_overlapped(left, right, parent->lightseg.left, parent->lightseg.right)) - { - //overlapped, return - return -1; - } - - if(before(parent->max, right)) - { - parent->max = right; - } - if(before(left, parent->lightseg.left)) - { - link = &parent->rb.rb_left; - } - else - { - link = &parent->rb.rb_right; - } - } - new_seg->max = right; - rb_link_node(&new_seg->rb, rb_parent, link); - rb_insert_augmented(&new_seg->rb, root, &__interval_tree_augment_callbacks); - - /* updata statistics */ - shadow_ivi->segs_cnt ++; - shadow_ivi->segs_length += seg->right - seg->left + 1; - shadow_ivi->mem_occupy += sizeof(IVI_shadow_seg_t); - return 0; -} - - - -/** - * Name: - * IVI_remove - * Description: - * Remove a given segment from given InterVal Index handler. - * Params: - * handler: The handler of InterVal Index created by IVI_create - * seg: A segment that user wants to delete. It MUST be created - * by IVI_seg_malloc. - * Return: - * On success, 0 is returned; - * Else when overlapp occures, -1 is returned. - **/ -int IVI_remove(IVI_t * handler, IVI_seg_t * seg) -{ - if(NULL == handler || NULL == seg) - { - return -1; - } - - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_root * root = &(shadow_ivi->root); - IVI_shadow_seg_t * new_seg = (IVI_shadow_seg_t *)seg; - rb_erase_augmented(&new_seg->rb, root, &__interval_tree_augment_callbacks); - - /* updata statistics */ - shadow_ivi->segs_cnt --; - shadow_ivi->segs_length -= seg->right - seg->left + 1; - shadow_ivi->mem_occupy -= sizeof(IVI_shadow_seg_t); - - return 0; -} - - - -static struct rb_node * __min_interval_search_from(struct rb_node * node, OFFSET_TYPE left, OFFSET_TYPE right) -{ - if(node == NULL) - { - return NULL; - } - IVI_shadow_seg_t * seg = rb_entry(node, IVI_shadow_seg_t, rb); - IVI_shadow_seg_t * left_seg = rb_entry(node->rb_left, IVI_shadow_seg_t, rb); - if(node->rb_left != NULL && !before(left_seg->max, left)) - { - struct rb_node * ret = __min_interval_search_from(node->rb_left, left, right); - if(ret != NULL) - { - return ret; - } - else if(__is_overlapped(left, right, seg->lightseg.left, seg->lightseg.right)) - { - return node; - } - else - { - return NULL; - } - } - else if(__is_overlapped(left, right, seg->lightseg.left, seg->lightseg.right)) - { - return node; - } - else - { - return __min_interval_search_from(node->rb_right, left, right); - } -} - - - -/** - * Name: - * IVI_query - * Description: - * Query from given InterVal Index and get the number of segments - * which are overlapped with given interval, and store those segments - * in the last parameter. - * Params: - * handler: The handler of interval index created by IVI_create - * left: Left point of given interval - * right: Right point of given interval - * segs: An address of a segment pointer array to store those segments which - * are overlapped with given interval. NOTE that user should not malloc - * the array, and segs need to be freed by user. The element of *segs - * MUST not be freed by user. - * Return: - * Return the number of segments which are overlapped with given interval - **/ -int IVI_query(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs) -{ - if(NULL == handler || after(left, right)) - { - //augments error - return -1; - } - - int interval_cnt = 0, max_cnt = 8; - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_node * root = shadow_ivi->root.rb_node; - struct rb_node * min_overlap = __min_interval_search_from(root, left, right); - struct rb_node * tmp_node = min_overlap; - - *segs = (IVI_seg_t **)malloc(max_cnt * sizeof(IVI_seg_t *)); - while (tmp_node != NULL) - { - IVI_seg_t * tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb)); - if(!__is_overlapped(tmp_seg->left, tmp_seg->right, left, right)) - { - break; - } - if(interval_cnt > max_cnt) - { - max_cnt *= 2; - *segs = (IVI_seg_t **)realloc(*segs, max_cnt * sizeof(IVI_seg_t *)); - } - (*segs)[interval_cnt] = tmp_seg; - interval_cnt ++; - tmp_node = rb_next(tmp_node); - } - return interval_cnt; -} - - - -/** - * Name: - * IVI_query_continuous - * Description: - * Query from interval index handler and get the number of continous segments - * which are overlapped with given interval. - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * left: Left point of given interval - * right: Right point of given interval - * segs: An address of a segment pointer array to store those segments which - * are overlapped with given interval. NOTE that user should not malloc - * the array, and segs need to be freed by user. The element of *segs - * MUST not be freed by user. - * Return: - * Return the number of continous segments which are overlapped with given interval - **/ -int IVI_query_continuous(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs) -{ - if(NULL == handler || after(left, right)) - { - //augments error - return -1; - } - - int interval_cnt = 0, max_cnt = 8; - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - struct rb_node * root = shadow_ivi->root.rb_node; - struct rb_node * min_overlap = __min_interval_search_from(root, left, right); - struct rb_node * tmp_node = min_overlap; - - *segs = (IVI_seg_t **)malloc(max_cnt * sizeof(IVI_seg_t *)); - while (tmp_node != NULL) - { - IVI_seg_t * tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb)); - if(!__is_overlapped(tmp_seg->left, tmp_seg->right, left, right)) - { - break; - } - if(interval_cnt > max_cnt) - { - max_cnt += 8; - *segs = (IVI_seg_t **)realloc(*segs, max_cnt * sizeof(IVI_seg_t *)); - } - (*segs)[interval_cnt] = tmp_seg; - interval_cnt ++; - tmp_node = rb_next(tmp_node); - IVI_seg_t * prev_tmp_seg = tmp_seg; - tmp_seg = (IVI_seg_t *)(rb_entry(tmp_node, IVI_shadow_seg_t, rb)); - if(!continuous(prev_tmp_seg->right, tmp_seg->left)) - { - break; - } - } - return interval_cnt; -} - - - -/** - * Name: - * IVI_seg_cnt - * Description: - * Get the count of segments in given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the count of segments in given interval index handler - **/ -int IVI_seg_cnt(IVI_t * handler) -{ - if(handler == NULL) - return -1; - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - return shadow_ivi->segs_cnt; -} - - - -/** - * Name: - * IVI_seg_len - * Description: - * Get the length of whole segments in given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the length of whole segments in given interval index handler - **/ -OFFSET_TYPE IVI_seg_length(IVI_t * handler) -{ - if(handler == NULL) - return -1; - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - return shadow_ivi->segs_length; -} - - -/** - * Name: - * IVI_mem_occupy - * Description: - * Get the memory occupy of given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the memory occupy of given interval index handler - **/ -unsigned long long IVI_mem_occupy(IVI_t * handler) -{ - if(handler == NULL) - return 0; - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - return shadow_ivi->mem_occupy; -} - - -static void __inorder_traverse(struct rb_node * root, IVI_callback_t cb, void * usr_para) -{ - if(root == NULL) - { - return; - } - - /* save first in case of root is freed in callback */ - struct rb_node * left_node = root->rb_left; - struct rb_node * right_node = root->rb_right; - __inorder_traverse(left_node, cb, usr_para); - IVI_seg_t * seg = (IVI_seg_t *)(rb_entry(root, IVI_shadow_seg_t, rb)); - cb(seg, usr_para); - __inorder_traverse(right_node, cb, usr_para); - return; -} - -/** - * Name: - * IVI_traverse - * Description: - * Traverse given InterVal Index and execute given callback function - * one time for each seg in InterVal Index. - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * IVI_callback_t: Callback function for user to define. - * usr_para: Parameter user want to pass to callback function. - * Return: - * void - **/ -void IVI_traverse(IVI_t * handler, IVI_callback_t cb, void * usr_para) -{ - if(NULL == handler || NULL == cb) - { - return; - } - - IVI_shadow_t * shadow_ivi = (IVI_shadow_t *)handler; - __inorder_traverse(shadow_ivi->root.rb_node, cb, usr_para); - return; -} diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp deleted file mode 100644 index 426decc..0000000 --- a/src/entry/json2iris.cpp +++ /dev/null @@ -1,1286 +0,0 @@ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "cJSON.h" -#include "hiredis.h" -#include "map_str2int.h" -#include "Maat_table_schema.h" -#include "Maat_rule_internal.h" -#include "Maat_utils.h" - -#define maat_json (module_name_str("MAAT_JSON")) - -const char* untitled_group_name="Untitled"; -const int json_version=1; -#define MAX_PATH_LINE 256 -#define MAX_COLUMN_NUM 32 -struct group_info_t -{ - int group_id; - char group_name[MAX_PATH_LINE]; -}; -struct iris_table_t -{ - char table_name[MAX_PATH_LINE]; - char table_path[MAX_PATH_LINE]; - int line_count; - enum MAAT_TABLE_TYPE table_type; - void* buff; - size_t write_pos; - size_t buff_sz; -}; -struct iris_description_t -{ - int group_cnt; - int region_cnt; - - char tmp_iris_dir[MAX_PATH_LINE]; - char tmp_iris_index_dir[MAX_PATH_LINE]; - char index_path[MAX_PATH_LINE]; - - struct iris_table_t* group_table; - struct iris_table_t* group2group_table; - struct iris_table_t* group2compile_table; - struct iris_table_t* compile_table; - MESA_htable_handle group_name_map; - MESA_htable_handle iris_table_map; - struct maat_kv_store* str2int_map; - redisContext *redis_write_ctx; - char* encrypt_key; - char* encrypt_algo; - FILE* idx_fp; -}; -struct traslate_command_t -{ - const char* json_string; - char* json_value; - int json_type; - int str2int_flag; - int empty_allowed; - const char* default_string; - int default_int; -}; -struct iris_table_t* query_table_info(iris_description_t* p_iris, const char* table_name, enum MAAT_TABLE_TYPE table_type) -{ - struct iris_table_t* table_info=NULL; - table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name)); - if(table_info==NULL) - { - table_info=ALLOC(struct iris_table_t, 1); - table_info->line_count=0; - table_info->table_type=table_type; - memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name)-1, strlen(table_name))); - snprintf(table_info->table_path,sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); - MESA_htable_add(p_iris->iris_table_map, (const unsigned char*)table_info->table_name, strlen(table_info->table_name), table_info); - } - return table_info; -} -void free_iris_table_info(void* p) -{ - struct iris_table_t* table=(struct iris_table_t*)p; - free(table->buff); - table->buff=NULL; - free(table); -} -static int get_group_seq(struct iris_description_t* iris_cfg) -{ - redisReply* data_reply=NULL; - int sequence=0; - if(iris_cfg->redis_write_ctx==NULL) - { - sequence=iris_cfg->group_cnt; - } - else - { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var); - sequence=(int)data_reply->integer-1; - freeReplyObject(data_reply); - data_reply=NULL; - } - iris_cfg->group_cnt++; - return sequence; -} -static int get_region_seq(struct iris_description_t* iris_cfg) -{ - redisReply* data_reply=NULL; - int sequence=0; - if(iris_cfg->redis_write_ctx==NULL) - { - sequence=iris_cfg->region_cnt; - } - else - { - data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_region_id_var); - sequence=(int)data_reply->integer-1; - freeReplyObject(data_reply); - data_reply=NULL; - } - iris_cfg->region_cnt++; - return sequence; -} - -int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) -{ - memset(iris_cfg,0,sizeof(struct iris_description_t)); - snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); - snprintf(iris_cfg->tmp_iris_index_dir,sizeof(iris_cfg->tmp_iris_index_dir),"%s_iris_tmp/index",json_file); - snprintf(iris_cfg->index_path,sizeof(iris_cfg->index_path),"%s/full_config_index.%010d",iris_cfg->tmp_iris_index_dir,json_version); - - iris_cfg->redis_write_ctx=redis_write_ctx; - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=1; - hargs.hash_slot_size = 1024; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; - hargs.recursive = 0; - hargs.data_free = free; - hargs.data_expire_with_condition = NULL; - - iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(iris_cfg->group_name_map, 0); - - hargs.data_free = free_iris_table_info; - iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); - - iris_cfg->str2int_map=maat_kv_store_new(); - - maat_kv_register(iris_cfg->str2int_map, "yes",1); - maat_kv_register(iris_cfg->str2int_map, "no",0); - - maat_kv_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); - maat_kv_register(iris_cfg->str2int_map, "ip_plus",TABLE_TYPE_IP_PLUS); - maat_kv_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); - maat_kv_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR); - maat_kv_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS); - maat_kv_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTERVAL); - maat_kv_register(iris_cfg->str2int_map, "interval",TABLE_TYPE_INTERVAL); - maat_kv_register(iris_cfg->str2int_map, "intval_plus",TABLE_TYPE_INTERVAL_PLUS); - maat_kv_register(iris_cfg->str2int_map, "interval_plus",TABLE_TYPE_INTERVAL_PLUS); - maat_kv_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST); - maat_kv_register(iris_cfg->str2int_map, "similar",TABLE_TYPE_SIMILARITY); - - - maat_kv_register(iris_cfg->str2int_map, "ipv4",4); - maat_kv_register(iris_cfg->str2int_map, "ipv6",6); - - maat_kv_register(iris_cfg->str2int_map, "double",0); - maat_kv_register(iris_cfg->str2int_map, "single",1); - - maat_kv_register(iris_cfg->str2int_map, "none",0); - maat_kv_register(iris_cfg->str2int_map, "and",1); - maat_kv_register(iris_cfg->str2int_map, "regex",2); - maat_kv_register(iris_cfg->str2int_map, "offset",3); - - maat_kv_register(iris_cfg->str2int_map, "sub",0); - maat_kv_register(iris_cfg->str2int_map, "right",1); - maat_kv_register(iris_cfg->str2int_map, "suffix",1); - maat_kv_register(iris_cfg->str2int_map, "left",2); - maat_kv_register(iris_cfg->str2int_map, "prefix",2); - maat_kv_register(iris_cfg->str2int_map, "complete",3); - maat_kv_register(iris_cfg->str2int_map, "exact",3); - - maat_kv_register(iris_cfg->str2int_map, "uncase plain",0); - maat_kv_register(iris_cfg->str2int_map, "hexbin",1); - maat_kv_register(iris_cfg->str2int_map, "case plain",2); - - iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); - iris_cfg->group2compile_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2COMPILE); - iris_cfg->group2group_table=query_table_info(iris_cfg, group2group_tn, TABLE_TYPE_GROUP2GROUP); - - if(encrypt_key && encrypt_algo) - { - iris_cfg->encrypt_key=_maat_strdup(encrypt_key); - iris_cfg->encrypt_algo=_maat_strdup(encrypt_algo); - } - - return 0; -} -void clear_iris_descriptor(struct iris_description_t *iris_cfg) -{ - if(iris_cfg->group_name_map!=NULL) - { - MESA_htable_destroy(iris_cfg->group_name_map, NULL); - } - if(iris_cfg->iris_table_map!=NULL) - { - MESA_htable_destroy(iris_cfg->iris_table_map, NULL); - } - maat_kv_store_free(iris_cfg->str2int_map); - free(iris_cfg->encrypt_algo); - free(iris_cfg->encrypt_key); - return; -} -int create_tmp_dir(struct iris_description_t *p) -{ - - - if((access(p->tmp_iris_dir,F_OK))<0) - { - if((mkdir(p->tmp_iris_dir, 0777)) < 0) - { - return -1; - } - } - if((access(p->tmp_iris_index_dir,F_OK))<0) - { - if((mkdir(p->tmp_iris_index_dir, 0777)) < 0) - { - return -1; - } - } - return 0; -} -int set_file_rulenum(const char* path,int rulenum,void* logger) -{ - FILE* fp=NULL; - if(rulenum==0) - { - fp=fopen(path,"w"); - } - else - { - fp=fopen(path,"r+"); - } - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s failed %s at set rule num.",path,strerror(errno)); - return -1; - } - fprintf(fp,"%010d\n",rulenum); - fclose(fp); - return 0; -} -int direct_write_rule(cJSON* json, struct maat_kv_store* str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) -{ - int i=0,ret=-1; - cJSON* item=NULL; - cJSON dummy; - char *p=NULL; - int int_value=0; - for(i=0;itype!=cmd[i].json_type) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "%s not defined or wrong format.",cmd[i].json_string); - ret=-1; - goto error_out; - } - if(cmd[i].str2int_flag==1) - { - p=item->valuestring; - ret=maat_kv_read(str2int, p, &int_value); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "%s's value %s is not valid format.",cmd[i].json_string,p); - free(p); - ret=-1; - goto error_out; - } - cmd[i].json_value=(char*)malloc(21);/* 2^64+1 can be represented in 21 chars. */ - snprintf(cmd[i].json_value,21, "%d", int_value); - - } - else - { - switch(item->type) - { - case cJSON_Number: - cmd[i].json_value=cJSON_Print(item); - break; - case cJSON_String: - cmd[i].json_value=ALLOC(char, strlen(item->valuestring)+1); - memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring)); - break; - default://impossible ,already checked - assert(0); - break; - } - } - } - for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); - table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); - } - table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); - table->line_count++; - ret=0; - -error_out: - for(i=0;istr2int_map,json_cmd, cmd_cnt, table, logger); -} -int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) -{ - struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt=0; - memset(json_cmd,0,sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string="region_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="group_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="addr_type"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].str2int_flag=1; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="saddr_format"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="mask"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="src_ip1"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="0.0.0.0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="src_ip2"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="255.255.255.255"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="sport_format"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="mask"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="src_port1"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="src_port2"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="65535"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="daddr_format"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="mask"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="dst_ip1"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="0.0.0.0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="dst_ip2"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="255.255.255.255"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="dport_format"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="mask"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="dst_port1"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="dst_port2"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="65535"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="protocol"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_int=0; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="direction"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].str2int_flag=1; - json_cmd[cmd_cnt].empty_allowed=1; - json_cmd[cmd_cnt].default_string="double"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="is_valid"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); - -} - -int write_expr_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) -{ - struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt=0; - memset(json_cmd,0,sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string="region_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="group_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - if(table->table_type==TABLE_TYPE_EXPR_PLUS) - { - json_cmd[cmd_cnt].json_string="district"; - json_cmd[cmd_cnt].json_type=cJSON_String; - cmd_cnt++; - } - - json_cmd[cmd_cnt].json_string="keywords"; - json_cmd[cmd_cnt].json_type=cJSON_String; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="expr_type"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].str2int_flag=1; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="match_method"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].str2int_flag=1; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="format"; - json_cmd[cmd_cnt].json_type=cJSON_String; - json_cmd[cmd_cnt].str2int_flag=1; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="is_valid"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); - -} -int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) -{ - struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt=0; - memset(json_cmd,0,sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string="region_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="group_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - if(table->table_type==TABLE_TYPE_INTERVAL_PLUS) - { - json_cmd[cmd_cnt].json_string="district"; - json_cmd[cmd_cnt].json_type=cJSON_String; - cmd_cnt++; - } - - json_cmd[cmd_cnt].json_string="low_boundary"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="up_boundary"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="is_valid"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); - -} -int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) -{ - struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt=0; - memset(json_cmd,0,sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string="region_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="group_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="raw_len"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="digest"; - json_cmd[cmd_cnt].json_type=cJSON_String; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="cfds_level"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="is_valid"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); - -} -int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) -{ - struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt=0; - memset(json_cmd,0,sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string="region_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="group_id"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="target"; - json_cmd[cmd_cnt].json_type=cJSON_String; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="threshold"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string="is_valid"; - json_cmd[cmd_cnt].json_type=cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); - -} - -int write_plugin_line(cJSON* plug_table_json, int sequence, iris_description_t* p_iris, void* logger) -{ - cJSON* item=NULL,*table_content=NULL, *each_line=NULL; - struct iris_table_t* table_info=NULL; - const char* table_name=NULL, *line_content=NULL; - int i=0, line_cnt=0; - - item=cJSON_GetObjectItem(plug_table_json, "table_name"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "The %d plugin_table's table_name not defined or format error.", sequence); - return -1; - } - table_name=item->valuestring; - table_info=query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); - table_content=cJSON_GetObjectItem(plug_table_json, "table_content"); - if(table_content==NULL||table_content->type!=cJSON_Array) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "%d plugin_table's table_content not defined or format error." - ,sequence); - return -1; - } - line_cnt=cJSON_GetArraySize(table_content); - - for(i=0;itype!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "plugin_table %s's line %d format error.",table_info->table_name,i+1); - continue; - } - line_content=each_line->valuestring; - table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); - table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); - table_info->line_count++; - } - return 0; -} -int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_description_t* p_iris, void* logger) -{ - cJSON* item=NULL,*table_content=NULL; - int ret=0; - int region_id=0; - const char* table_name=NULL,*table_type_str=NULL; - enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR; - struct iris_table_t* table_info=NULL; - - - item=cJSON_GetObjectItem(region_json,"table_type"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d table name %s's table_type not defined or format error." - ,compile_id,table_name); - return -1; - } - table_type_str=item->valuestring; - ret=maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); - if(ret!=1) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d table name %s's table_type %s invalid." - ,compile_id,table_name,table_type_str); - return -1; - } - item=cJSON_GetObjectItem(region_json,"table_name"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's region table_name not defined or format error.",compile_id); - return -1; - } - table_name=item->valuestring; - table_info=query_table_info(p_iris, table_name, table_type); - - table_content=cJSON_GetObjectItem(region_json,"table_content"); - if(table_content==NULL||table_content->type!=cJSON_Object) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d table name %s's table_content not defined or format error." - ,compile_id,table_name); - return -1; - } - - region_id=get_region_seq(p_iris); - cJSON_AddNumberToObject(table_content, "region_id", region_id); - cJSON_AddNumberToObject(table_content, "group_id", group_id); - cJSON_AddNumberToObject(table_content, "is_valid", 1); - - switch(table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_IP: - ret=write_ip_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_IP_PLUS: - write_ip_plus_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_INTERVAL: - case TABLE_TYPE_INTERVAL_PLUS: - ret=write_intval_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_DIGEST: - ret=write_digest_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_SIMILARITY: - ret=write_similar_line(table_content, p_iris, table_info, logger); - break; - default: - assert(0); - break; - } - return ret; -} - -int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger) -{ - int compile_id=-1,cmd_cnt=0,ret=-1, clause_num=0, group_num=0, i=0, clause_index=0; - cJSON* item=NULL; - struct iris_table_t* table_info=NULL; - cJSON* group_array=NULL, *group_obj=NULL; - int* clause_ids=NULL; - - item=cJSON_GetObjectItem(compile,"compile_id"); - if(item->type!=cJSON_Number) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile_id format not number."); - return -1; - } - compile_id=item->valueint; - - group_array=cJSON_GetObjectItem(compile, "groups"); - group_num=cJSON_GetArraySize(group_array); - clause_ids=ALLOC(int, group_num); - cJSON_ArrayForEach(group_obj, group_array) - { - item=cJSON_GetObjectItem(group_obj, "clause_index"); - if(item) - { - clause_index=item->valueint; - for(i=0; itype!=cJSON_String) - { - table_info=p_iris->compile_table; - } - else - { - table_info=query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); - } - - ret=direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger); - if(ret<0) - { - return -1; - } - return compile_id; -} -int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index, const char* virtual_table, struct iris_description_t *p_iris, void * logger) -{ - char buff[1024*4]; - struct iris_table_t* table=p_iris->group2compile_table; - snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, clause_index); - table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); - table->line_count++; - return 0; -} -int write_group2group_line(int group_id, int superior_gorup_id, struct iris_description_t *p_iris, void * logger) -{ - char buff[1024*4]; - struct iris_table_t* table=p_iris->group2group_table; - snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_gorup_id); - table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); - table->line_count++; - return 0; -} - -void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) -{ - struct iris_description_t *p_iris=(struct iris_description_t *)user; - struct iris_table_t* table=(struct iris_table_t*)data; - FILE* table_fp=NULL; - char line_cnt_str[32], err_str[256]; - snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count); - - UNUSED int ret=0; - size_t table_file_sz=strlen(line_cnt_str)+table->write_pos; - unsigned char* buff=ALLOC(unsigned char, table_file_sz); - unsigned char* encrypt_buff=NULL; - size_t encrypt_buff_sz=0; - memcpy(buff, line_cnt_str, strlen(line_cnt_str)); - memcpy(buff+strlen(line_cnt_str), table->buff, table->write_pos); - table_fp=fopen(table->table_path, "w"); - if(p_iris->encrypt_key) - { - ret=crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); - assert(ret==0); - fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); - fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); - free(encrypt_buff); - } - else - { - fwrite(buff, table_file_sz, 1, table_fp); - fprintf(p_iris->idx_fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); - } - fclose(table_fp); - free(buff); - buff=NULL; - - -} -int write_index_file(struct iris_description_t *p_iris,void* logger) -{ - p_iris->idx_fp=fopen(p_iris->index_path,"w"); - if(p_iris->idx_fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); - return -1; - } - MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, p_iris); - fclose(p_iris->idx_fp); - p_iris->idx_fp=NULL; - return 0; -} -static struct group_info_t* group_info_read(MESA_htable_handle table, const char* group_name) -{ - return (struct group_info_t*)MESA_htable_search(table, (const unsigned char*)group_name, strlen(group_name)); -} -static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_iris, MESA_htable_handle table, const char* group_name) -{ - static struct group_info_t untitled_group; - struct group_info_t *group_info=NULL; - if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) - { - group_info=&untitled_group; - group_info->group_id=get_group_seq(p_iris); - } - else - { - group_info=ALLOC(struct group_info_t, 1); - group_info->group_id=get_group_seq(p_iris); - strncpy(group_info->group_name, group_name, sizeof(group_info->group_name)); - MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); - } - return group_info; -} - -int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, int Nth_group, struct iris_description_t *p_iris, void* logger) -{ - const char* _str_parent_type[2]={"compile", "group"}; - int ret=0, i=0; - int group_not_flag=0, clause_index=0; - cJSON *region_json=NULL, *item=NULL; - cJSON *sub_groups=NULL, *region_rule=NULL; - const char* group_name=NULL, *virtual_table=NULL; - struct group_info_t *group_info=NULL; - - item=cJSON_GetObjectItem(group_json, "group_name"); - if(item==NULL||item->type!=cJSON_String) - { - group_name=untitled_group_name; - } - else - { - group_name=item->valuestring; - } - if(parent_type==PARENT_TYPE_COMPILE) - { - item=cJSON_GetObjectItem(group_json, "virtual_table"); - if(item==NULL||item->type!=cJSON_String) - { - virtual_table="null"; - } - else - { - virtual_table=item->valuestring; - } - item=cJSON_GetObjectItem(group_json,"not_flag"); - if(item==NULL||item->type!=cJSON_Number) - { - group_not_flag=0; - } - else - { - group_not_flag=item->valueint; - } - item=cJSON_GetObjectItem(group_json,"clause_index"); - if(item==NULL||item->type!=cJSON_Number) - { - clause_index=Nth_group; - } - else - { - clause_index=item->valueint; - } - } - else - { - group_not_flag=0; - } - group_info=group_info_read(p_iris->group_name_map, group_name); - if(group_info==NULL)//exist group name, regions and sub groups will be ommit. - { - group_info=group_info_add_unsafe(p_iris, p_iris->group_name_map, group_name); - region_json=cJSON_GetObjectItem(group_json,"regions"); - if(region_json!=NULL) - { - cJSON_ArrayForEach(region_rule, region_json) - { - ret=write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "compile rule %d write region error.", tracking_compile_id); - return -1; - } - } - } - sub_groups=cJSON_GetObjectItem(group_json,"sub_groups"); - if(sub_groups!=NULL) - { - //recursively - i=0; - cJSON_ArrayForEach(item, sub_groups) - { - i++; - ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger); - if(ret<0) - { - return -1; - } - - } - } - if(region_json==NULL && sub_groups==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_json, - "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group.", tracking_compile_id); - } - } - if(parent_type==PARENT_TYPE_COMPILE) - { - ret=write_group2compile_line(group_info->group_id, parent_id, group_not_flag, clause_index, virtual_table, p_iris, logger); - } - else - { - ret=write_group2group_line(group_info->group_id, parent_id, p_iris, logger); - } - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "%s rule %d write group error.", _str_parent_type[parent_type], parent_id); - return -1; - } - - return 0; -} -int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) -{ - int i=0; - int compile_id=-1, compile_cnt=0, group_cnt=0; - int ret=0; - cJSON *compile_array=NULL, *group_array=NULL, *plug_tables=NULL; - cJSON *compile_obj=NULL, *group_obj=NULL, *each_plug_table=NULL, *item=NULL; - static struct group_info_t* parent_group=NULL; - const char* parent_group_name=NULL; - - plug_tables=cJSON_GetObjectItem(json, "plugin_table"); - if(NULL!=plug_tables) - { - i=0; - cJSON_ArrayForEach(each_plug_table, plug_tables) - { - write_plugin_line(each_plug_table, i, p_iris, logger); - i++; - } - } - group_array=cJSON_GetObjectItem(json, "groups");//sub-group to group - if(group_array!=NULL) - { - cJSON_ArrayForEach(group_obj, group_array) - { - item=cJSON_GetObjectItem(group_obj, "parent_group"); - if(item==NULL || item->type!=cJSON_String) - { - parent_group_name=untitled_group_name; - - } - else - { - parent_group_name=item->string; - } - parent_group=group_info_read(p_iris->group_name_map, parent_group_name); - if(parent_group==NULL) - { - parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, parent_group_name); - } - ret=write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger); - if(ret<0) - { - return -1; - } - } - - } - compile_cnt=0; - compile_array=cJSON_GetObjectItem(json,"rules"); - if(compile_array!=NULL) - { - compile_cnt=cJSON_GetArraySize(compile_array); - } - if(compile_cnt>0) - { - cJSON_ArrayForEach(compile_obj, compile_array) - { - compile_id=write_compile_line(compile_obj,p_iris, logger); - if(compile_id<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "In %d compile rule.", i); - return -1; - } - group_array=cJSON_GetObjectItem(compile_obj, "groups"); - if(group_array==NULL) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "compile rule %d have no group.",compile_id); - return -1; - } - group_cnt=cJSON_GetArraySize(group_array); - if(group_cnt<=0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "compile rule %d have no groups.",compile_id); - return -1; - } - i=0; - cJSON_ArrayForEach(group_obj, group_array) - { - ret=write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger); - if(ret<0) - { - return -1; - } - i++; - } - } - } - ret=write_index_file(p_iris, logger); - if(ret<0) - { - return -1; - } - return 0; -} -// redis_write_ctx is used by maat_redis_tool to write json to redis. -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger) -{ - cJSON *json=NULL, *tmp_obj=NULL; - int ret=-1; - struct iris_description_t iris_cfg; - memset(&iris_cfg, 0, sizeof(iris_cfg)); - json=cJSON_Parse(json_buff); - if (!json) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr()); - goto error_out; - } - tmp_obj=cJSON_GetObjectItem(json, "compile_table"); - if(tmp_obj) - { - compile_tn=tmp_obj->valuestring; - } - - tmp_obj=cJSON_GetObjectItem(json, "group2compile_table"); - if(tmp_obj) - { - group2compile_tn=tmp_obj->valuestring; - } - tmp_obj=cJSON_GetObjectItem(json, "group2group_table"); - if(tmp_obj) - { - group2group_tn=tmp_obj->valuestring; - } - ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group2compile_tn, group2group_tn, redis_write_ctx, &iris_cfg, logger); - if(ret<0) - { - goto error_out; - } - ret=create_tmp_dir(&iris_cfg); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, - "create tmp folder %s error", iris_cfg.tmp_iris_dir); - goto error_out; - } - ret=write_iris(json, &iris_cfg, logger); - if(ret<0) - { - goto error_out; - } - memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); - - cJSON_Delete(json); - clear_iris_descriptor(&iris_cfg); - return 0; - - -error_out: - cJSON_Delete(json); - clear_iris_descriptor(&iris_cfg); - return -1; -} - diff --git a/src/entry/map_str2int.cpp b/src/entry/map_str2int.cpp deleted file mode 100644 index a4265f7..0000000 --- a/src/entry/map_str2int.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include -#include "Maat_utils.h" -#include "uthash/uthash.h" - -const size_t MAAT_KV_MAX_KEY_LEN=512; - -struct maat_kv_pair -{ - char* key; //must be lower case. - size_t keylen; - int val; - UT_hash_handle hh; -}; -void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_sz) -{ - size_t i=0; - for(i=0; ikeylen=keylen; - kv->val=value; - kv->key=ALLOC(char, keylen); - strlowercase(key, keylen, kv->key, kv->keylen); - return kv; -} -void maat_kv_pair_free(struct maat_kv_pair* kv) -{ - free(kv->key); - kv->key=NULL; - free(kv); -} -struct maat_kv_store -{ - struct maat_kv_pair* hash; -}; - -struct maat_kv_store* maat_kv_store_new(void) -{ - - struct maat_kv_store* store=ALLOC(struct maat_kv_store, 1); - return store; -} -void maat_kv_store_free(struct maat_kv_store* store) -{ - struct maat_kv_pair* kv=NULL, *tmp_kv=NULL; - if(store==NULL) return; - HASH_ITER(hh, store->hash, kv, tmp_kv) - { - HASH_DEL(store->hash, kv); - maat_kv_pair_free(kv); - } - free(store); - return; -} -int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value) -{ - struct maat_kv_pair *kv=NULL; - struct maat_kv_pair *tmp_kv=NULL; - if(keylen>MAAT_KV_MAX_KEY_LEN) - { - return -1; - } - kv=maat_kv_pair_new(key, keylen, value); - HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv); - if(tmp_kv) - { - maat_kv_pair_free(kv); - return -1; - } - HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); - return 1; -} - -int maat_kv_register(struct maat_kv_store* store, const char* key, int value) -{ - int ret=0; - ret=maat_kv_register_unNull(store, key, strlen(key), value); - return ret; -} - -int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int* value) -{ - struct maat_kv_pair *kv=NULL; - char key_lowercase[MAAT_KV_MAX_KEY_LEN]={0}; - if(keylen>MAAT_KV_MAX_KEY_LEN) - { - return -1; - } - strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); - HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); - if(kv) - { - *value=kv->val; - return 1; - } - else - { - return -1; - } -} - -int maat_kv_read(struct maat_kv_store * store, const char * key, int * value) -{ - return maat_kv_read_unNull(store, key, strlen(key), value); -} -struct maat_kv_store* maat_kv_store_duplicate(struct maat_kv_store* origin_map) -{ - struct maat_kv_store* target=maat_kv_store_new(); - struct maat_kv_pair* kv=NULL, *tmp_kv=NULL, *copy_kv=NULL; - HASH_ITER(hh, origin_map->hash, kv, tmp_kv) - { - copy_kv=maat_kv_pair_new(kv->key, kv->keylen, kv->val); - HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->keylen, copy_kv); - } - return target; -} - diff --git a/src/entry/rbtree.c b/src/entry/rbtree.c deleted file mode 100644 index 4c68ad1..0000000 --- a/src/entry/rbtree.c +++ /dev/null @@ -1,548 +0,0 @@ -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - (C) 2002 David Woodhouse - (C) 2012 Michel Lespinasse - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/lib/rbtree.c -*/ - -#include "rbtree.h" -#include "rbtree_augmented.h" -/* - * red-black trees properties: http://en.wikipedia.org/wiki/Rbtree - * - * 1) A node is either red or black - * 2) The root is black - * 3) All leaves (NULL) are black - * 4) Both children of every red node are black - * 5) Every simple path from root to leaves contains the same number - * of black nodes. - * - * 4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two - * consecutive red nodes in a path and every red node is therefore followed by - * a black. So if B is the number of black nodes on every simple path (as per - * 5), then the longest possible path due to 4 is 2B. - * - * We shall indicate color with case, where black nodes are uppercase and red - * nodes will be lowercase. Unknown color nodes shall be drawn as red within - * parentheses and have some accompanying text comment. - */ - -static inline void rb_set_black(struct rb_node *rb) -{ - rb->__rb_parent_color |= RB_BLACK; -} - -static inline struct rb_node *rb_red_parent(struct rb_node *red) -{ - return (struct rb_node *)red->__rb_parent_color; -} - -/* - * Helper function for rotations: - * - old's parent and color get assigned to new - * - old gets assigned new as a parent and 'color' as a color. - */ -static inline void -__rb_rotate_set_parents(struct rb_node *old, struct rb_node *new, - struct rb_root *root, int color) -{ - struct rb_node *parent = rb_parent(old); - new->__rb_parent_color = old->__rb_parent_color; - rb_set_parent_color(old, new, color); - __rb_change_child(old, new, parent, root); -} - -static __always_inline void -__rb_insert(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - struct rb_node *parent = rb_red_parent(node), *gparent, *tmp; - - while (1) { - /* - * Loop invariant: node is red - * - * If there is a black parent, we are done. - * Otherwise, take some corrective action as we don't - * want a red root or two consecutive red nodes. - */ - if (!parent) { - rb_set_parent_color(node, NULL, RB_BLACK); - break; - } else if (rb_is_black(parent)) - break; - - gparent = rb_red_parent(parent); - - tmp = gparent->rb_right; - if (parent != tmp) { /* parent == gparent->rb_left */ - if (tmp && rb_is_red(tmp)) { - /* - * Case 1 - color flips - * - * G g - * / \ / \ - * p u --> P U - * / / - * n n - * - * However, since g's parent might be red, and - * 4) does not allow this, we need to recurse - * at g. - */ - rb_set_parent_color(tmp, gparent, RB_BLACK); - rb_set_parent_color(parent, gparent, RB_BLACK); - node = gparent; - parent = rb_parent(node); - rb_set_parent_color(node, parent, RB_RED); - continue; - } - - tmp = parent->rb_right; - if (node == tmp) { - /* - * Case 2 - left rotate at parent - * - * G G - * / \ / \ - * p U --> n U - * \ / - * n p - * - * This still leaves us in violation of 4), the - * continuation into Case 3 will fix that. - */ - parent->rb_right = tmp = node->rb_left; - node->rb_left = parent; - if (tmp) - rb_set_parent_color(tmp, parent, - RB_BLACK); - rb_set_parent_color(parent, node, RB_RED); - augment_rotate(parent, node); - parent = node; - tmp = node->rb_right; - } - - /* - * Case 3 - right rotate at gparent - * - * G P - * / \ / \ - * p U --> n g - * / \ - * n U - */ - gparent->rb_left = tmp; /* == parent->rb_right */ - parent->rb_right = gparent; - if (tmp) - rb_set_parent_color(tmp, gparent, RB_BLACK); - __rb_rotate_set_parents(gparent, parent, root, RB_RED); - augment_rotate(gparent, parent); - break; - } else { - tmp = gparent->rb_left; - if (tmp && rb_is_red(tmp)) { - /* Case 1 - color flips */ - rb_set_parent_color(tmp, gparent, RB_BLACK); - rb_set_parent_color(parent, gparent, RB_BLACK); - node = gparent; - parent = rb_parent(node); - rb_set_parent_color(node, parent, RB_RED); - continue; - } - - tmp = parent->rb_left; - if (node == tmp) { - /* Case 2 - right rotate at parent */ - parent->rb_left = tmp = node->rb_right; - node->rb_right = parent; - if (tmp) - rb_set_parent_color(tmp, parent, - RB_BLACK); - rb_set_parent_color(parent, node, RB_RED); - augment_rotate(parent, node); - parent = node; - tmp = node->rb_left; - } - - /* Case 3 - left rotate at gparent */ - gparent->rb_right = tmp; /* == parent->rb_left */ - parent->rb_left = gparent; - if (tmp) - rb_set_parent_color(tmp, gparent, RB_BLACK); - __rb_rotate_set_parents(gparent, parent, root, RB_RED); - augment_rotate(gparent, parent); - break; - } - } -} - -/* - * Inline version for rb_erase() use - we want to be able to inline - * and eliminate the dummy_rotate callback there - */ -static __always_inline void -____rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - struct rb_node *node = NULL, *sibling, *tmp1, *tmp2; - - while (1) { - /* - * Loop invariants: - * - node is black (or NULL on first iteration) - * - node is not the root (parent is not NULL) - * - All leaf paths going through parent and node have a - * black node count that is 1 lower than other leaf paths. - */ - sibling = parent->rb_right; - if (node != sibling) { /* node == parent->rb_left */ - if (rb_is_red(sibling)) { - /* - * Case 1 - left rotate at parent - * - * P S - * / \ / \ - * N s --> p Sr - * / \ / \ - * Sl Sr N Sl - */ - parent->rb_right = tmp1 = sibling->rb_left; - sibling->rb_left = parent; - rb_set_parent_color(tmp1, parent, RB_BLACK); - __rb_rotate_set_parents(parent, sibling, root, - RB_RED); - augment_rotate(parent, sibling); - sibling = tmp1; - } - tmp1 = sibling->rb_right; - if (!tmp1 || rb_is_black(tmp1)) { - tmp2 = sibling->rb_left; - if (!tmp2 || rb_is_black(tmp2)) { - /* - * Case 2 - sibling color flip - * (p could be either color here) - * - * (p) (p) - * / \ / \ - * N S --> N s - * / \ / \ - * Sl Sr Sl Sr - * - * This leaves us violating 5) which - * can be fixed by flipping p to black - * if it was red, or by recursing at p. - * p is red when coming from Case 1. - */ - rb_set_parent_color(sibling, parent, - RB_RED); - if (rb_is_red(parent)) - rb_set_black(parent); - else { - node = parent; - parent = rb_parent(node); - if (parent) - continue; - } - break; - } - /* - * Case 3 - right rotate at sibling - * (p could be either color here) - * - * (p) (p) - * / \ / \ - * N S --> N Sl - * / \ \ - * sl Sr s - * \ - * Sr - */ - sibling->rb_left = tmp1 = tmp2->rb_right; - tmp2->rb_right = sibling; - parent->rb_right = tmp2; - if (tmp1) - rb_set_parent_color(tmp1, sibling, - RB_BLACK); - augment_rotate(sibling, tmp2); - tmp1 = sibling; - sibling = tmp2; - } - /* - * Case 4 - left rotate at parent + color flips - * (p and sl could be either color here. - * After rotation, p becomes black, s acquires - * p's color, and sl keeps its color) - * - * (p) (s) - * / \ / \ - * N S --> P Sr - * / \ / \ - * (sl) sr N (sl) - */ - parent->rb_right = tmp2 = sibling->rb_left; - sibling->rb_left = parent; - rb_set_parent_color(tmp1, sibling, RB_BLACK); - if (tmp2) - rb_set_parent(tmp2, parent); - __rb_rotate_set_parents(parent, sibling, root, - RB_BLACK); - augment_rotate(parent, sibling); - break; - } else { - sibling = parent->rb_left; - if (rb_is_red(sibling)) { - /* Case 1 - right rotate at parent */ - parent->rb_left = tmp1 = sibling->rb_right; - sibling->rb_right = parent; - rb_set_parent_color(tmp1, parent, RB_BLACK); - __rb_rotate_set_parents(parent, sibling, root, - RB_RED); - augment_rotate(parent, sibling); - sibling = tmp1; - } - tmp1 = sibling->rb_left; - if (!tmp1 || rb_is_black(tmp1)) { - tmp2 = sibling->rb_right; - if (!tmp2 || rb_is_black(tmp2)) { - /* Case 2 - sibling color flip */ - rb_set_parent_color(sibling, parent, - RB_RED); - if (rb_is_red(parent)) - rb_set_black(parent); - else { - node = parent; - parent = rb_parent(node); - if (parent) - continue; - } - break; - } - /* Case 3 - right rotate at sibling */ - sibling->rb_right = tmp1 = tmp2->rb_left; - tmp2->rb_left = sibling; - parent->rb_left = tmp2; - if (tmp1) - rb_set_parent_color(tmp1, sibling, - RB_BLACK); - augment_rotate(sibling, tmp2); - tmp1 = sibling; - sibling = tmp2; - } - /* Case 4 - left rotate at parent + color flips */ - parent->rb_left = tmp2 = sibling->rb_right; - sibling->rb_right = parent; - rb_set_parent_color(tmp1, sibling, RB_BLACK); - if (tmp2) - rb_set_parent(tmp2, parent); - __rb_rotate_set_parents(parent, sibling, root, - RB_BLACK); - augment_rotate(parent, sibling); - break; - } - } -} - -/* Non-inline version for rb_erase_augmented() use */ -void __rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - ____rb_erase_color(parent, root, augment_rotate); -} - -/* - * Non-augmented rbtree manipulation functions. - * - * We use dummy augmented callbacks here, and have the compiler optimize them - * out of the rb_insert_color() and rb_erase() function definitions. - */ - -static inline void dummy_propagate(struct rb_node *node, struct rb_node *stop) {} -static inline void dummy_copy(struct rb_node *old, struct rb_node *new) {} -static inline void dummy_rotate(struct rb_node *old, struct rb_node *new) {} - -static const struct rb_augment_callbacks dummy_callbacks = { - dummy_propagate, dummy_copy, dummy_rotate -}; - -void rb_insert_color(struct rb_node *node, struct rb_root *root) -{ - __rb_insert(node, root, dummy_rotate); -} - -void rb_erase(struct rb_node *node, struct rb_root *root) -{ - struct rb_node *rebalance; - rebalance = __rb_erase_augmented(node, root, &dummy_callbacks); - if (rebalance) - ____rb_erase_color(rebalance, root, dummy_rotate); -} - -/* - * Augmented rbtree manipulation functions. - * - * This instantiates the same __always_inline functions as in the non-augmented - * case, but this time with user-defined callbacks. - */ - -void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)) -{ - __rb_insert(node, root, augment_rotate); -} - -/* - * This function returns the first node (in sort order) of the tree. - */ -struct rb_node *rb_first(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_left) - n = n->rb_left; - return n; -} - -struct rb_node *rb_last(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_right) - n = n->rb_right; - return n; -} - -struct rb_node *rb_next(const struct rb_node *node) -{ - struct rb_node *parent; - - if (RB_EMPTY_NODE(node)) - return NULL; - - /* - * If we have a right-hand child, go down and then left as far - * as we can. - */ - if (node->rb_right) { - node = node->rb_right; - while (node->rb_left) - node=node->rb_left; - return (struct rb_node *)node; - } - - /* - * No right-hand children. Everything down and left is smaller than us, - * so any 'next' node must be in the general direction of our parent. - * Go up the tree; any time the ancestor is a right-hand child of its - * parent, keep going up. First time it's a left-hand child of its - * parent, said parent is our 'next' node. - */ - while ((parent = rb_parent(node)) && node == parent->rb_right) - node = parent; - - return parent; -} - -struct rb_node *rb_prev(const struct rb_node *node) -{ - struct rb_node *parent; - - if (RB_EMPTY_NODE(node)) - return NULL; - - /* - * If we have a left-hand child, go down and then right as far - * as we can. - */ - if (node->rb_left) { - node = node->rb_left; - while (node->rb_right) - node=node->rb_right; - return (struct rb_node *)node; - } - - /* - * No left-hand children. Go up till we find an ancestor which - * is a right-hand child of its parent. - */ - while ((parent = rb_parent(node)) && node == parent->rb_left) - node = parent; - - return parent; -} - -void rb_replace_node(struct rb_node *victim, struct rb_node *new, - struct rb_root *root) -{ - struct rb_node *parent = rb_parent(victim); - - /* Set the surrounding nodes to point to the replacement */ - __rb_change_child(victim, new, parent, root); - if (victim->rb_left) - rb_set_parent(victim->rb_left, new); - if (victim->rb_right) - rb_set_parent(victim->rb_right, new); - - /* Copy the pointers/colour from the victim to the replacement */ - *new = *victim; -} - -static struct rb_node *rb_left_deepest_node(const struct rb_node *node) -{ - for (;;) { - if (node->rb_left) - node = node->rb_left; - else if (node->rb_right) - node = node->rb_right; - else - return (struct rb_node *)node; - } -} - -struct rb_node *rb_next_postorder(const struct rb_node *node) -{ - const struct rb_node *parent; - if (!node) - return NULL; - parent = rb_parent(node); - - /* If we're sitting on node, we've already seen our children */ - if (parent && node == parent->rb_left && parent->rb_right) { - /* If we are the parent's left node, go to the parent's right - * node then all the way down to the left */ - return rb_left_deepest_node(parent->rb_right); - } else - /* Otherwise we are the parent's right node, and the parent - * should be next */ - return (struct rb_node *)parent; -} - -struct rb_node *rb_first_postorder(const struct rb_root *root) -{ - if (!root->rb_node) - return NULL; - - return rb_left_deepest_node(root->rb_node); -} diff --git a/src/entry/stream_fuzzy_hash.c b/src/entry/stream_fuzzy_hash.c deleted file mode 100644 index 5adf5b9..0000000 --- a/src/entry/stream_fuzzy_hash.c +++ /dev/null @@ -1,737 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "sfh_internal.h" -#include "stream_fuzzy_hash.h" -#include "interval_index.h" -//#define DEBUG_PRINT -#define INIT_SIZE 128 -#define ENTROPY_THRESHOLD 0.5 -#define MULTIPLE 4 -int count = 0; -const char * map_to64bytes = -"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - - -double get_rs_entropy(unsigned int * r_array, unsigned int r_index); -int cmp(const void * a, const void * b); -void sfh_rs_entropy(IVI_seg_t * seg, void * user_para); -void sfh_output_state_t(IVI_seg_t * seg, void * user_para); -int write_uint_array(unsigned int ** array, unsigned int *index,unsigned int *size,unsigned int value); -/** - * roll_state³õʼ»¯ - */ -static inline void roll_init(struct roll_state_t * self) -{ - memset(self, 0, sizeof(struct roll_state_t)); -} - -/** - * ¼ÆËãroll_hashÖµ£¬½«ÍⲿÊý¾Ý¶ÁÈ¡µ½´°¿ÚÖÐ - */ -static inline void roll_hash(struct roll_state_t * self, unsigned char c) -{ - self->h2 -= self->h1; - self->h2 += ROLLING_WINDOW * (unsigned int)c; - - self->h1 += (unsigned int)c; - self->h1 -= (unsigned int)self->window[self->n]; - - self->window[self->n] = c; - self->n++; - if (self->n == ROLLING_WINDOW) - { - self->n = 0; - } - self->h3 <<= 5; - self->h3 ^= c; -} - -/** - * ¼ÆËã´°¿ÚÀïÃæµÄroll_hashÖµ£¬Ã¿´Îroll_hashÖµÂú×ãÒ»¶¨Ìõ¼þ£¬·ÖƬ - */ -static inline unsigned int roll_sum(const struct roll_state_t * self) -{ - return self->h1 + self->h2 + self->h3; -} - -/** - * ¼ÆËã·ÖƬµÄFNVÖµ - */ -static inline unsigned int sum_hash(unsigned char c, unsigned int h) -{ - return (h * HASH_PRIME) ^ c; -} - -/** - * ´´½¨handle - */ -sfh_instance_t * SFH_instance(unsigned long long origin_len) -{ - fuzzy_handle_inner_t * handle = NULL; - unsigned long long tmp_blksize = 0; - tmp_blksize = get_blocksize(origin_len); - - if(tmp_blksize==0) - { - return NULL; - } - handle = (fuzzy_handle_inner_t *)calloc(1,sizeof(fuzzy_handle_inner_t)); - handle->fuzzy_node_memory = 0; - handle->IVI_memory = 0; - handle->fuzzy_node_memory += sizeof(fuzzy_handle_inner_t); - handle->orilen = origin_len; - handle->ivi = IVI_create(); - handle->effective_length = 0; - handle->length_increase = 0; - handle->sim_tuned_rs_cnt = 0; - //handle->blocksize=tmp_blksize; - handle->blocksize = 3; - handle->do_tune=1; - return (sfh_instance_t *)handle; -} - - -/** - * IVI_destroyµÄ»Øµ÷º¯Êý£¬Ïú»ÙIVIÖеÄÊý¾Ý - */ -void fuzzy_node_free(IVI_seg_t * seg, void * usr_para) -{ - fuzzy_handle_inner_t * _handle = (fuzzy_handle_inner_t *)usr_para; - sfh_seg_t * temp = (sfh_seg_t*)(seg->data); - _handle->fuzzy_node_memory-=destroy_sfh_seg(temp); - return; -} - - -void SFH_release(sfh_instance_t * handle) -{ - IVI_destroy(((fuzzy_handle_inner_t *)handle)->ivi, fuzzy_node_free, (void *)handle); - ((fuzzy_handle_inner_t *)handle)->fuzzy_node_memory -= sizeof(fuzzy_handle_inner_t); - free((fuzzy_handle_inner_t *)handle); - return; -} -void sfh_tune_simulation(IVI_seg_t * seg, void * user_para) -{ - sfh_seg_t * tmp = (sfh_seg_t *)(seg->data); - int i = 0; - fuzzy_handle_inner_t * _handle = (fuzzy_handle_inner_t *)user_para; - unsigned long long blocksize = _handle->blocksize * MULTIPLE; - for(i = 0; i < tmp->r_cnt; i++) - { - if(tmp->r_array[i] % blocksize == blocksize -1) - { - _handle->sim_tuned_rs_cnt ++; - } - } -} -void sfh_tune_seg(sfh_seg_t * p, unsigned long long blocksize) -{ - int i = 0, j = 0; - struct zt_state_t tmp_zt; - int new_zt_cnt=0; - zt_hash_initial(&tmp_zt); - - for(j = 0; j < p->r_cnt; j++) - { - if(j == 0) - { - zt_hash_arymul(&tmp_zt, &(p->p_state)); - } - else - { - zt_hash_arymul(&tmp_zt, &(p->s_array[j - 1])); - } - if(p->r_array[j] % blocksize == blocksize - 1) - { - p->r_array[i]=p->r_array[j]; - i++; - if(i>1) - { - p->s_array[new_zt_cnt].val=tmp_zt.val; - new_zt_cnt++; - } - else - { - p->p_state.val=tmp_zt.val; - } - zt_hash_initial(&tmp_zt); - } - } - zt_hash_arymul(&tmp_zt, &(p->s_state)); - if(i == 0) - { - zt_hash_initial(&(p->p_state)); - } - p->s_state.val = tmp_zt.val; - p->s_cnt = new_zt_cnt; - p->r_cnt = i; - assert(p->r_cnt>=p->s_cnt); -} -void sfh_tune_callback(IVI_seg_t * seg, void * user_para) -{ - sfh_seg_t * p = (sfh_seg_t *)(seg->data); - if(p->r_cnt== 0) - { - return; - } - - fuzzy_handle_inner_t * _handle = (fuzzy_handle_inner_t *)user_para; - unsigned long long blocksize = _handle->blocksize; - _handle->s_state_cnt-=p->s_cnt; - sfh_tune_seg(p, blocksize); - _handle->s_state_cnt+=p->s_cnt; - //printf("after state_cnt:%d,block:%llu\n",_handle->s_state_cnt,_handle->blocksize); -} - -void do_sfh_tune(sfh_instance_t * handle) -{ - fuzzy_handle_inner_t * _handle=(fuzzy_handle_inner_t *)handle; - do{ - _handle->sim_tuned_rs_cnt = 0; - IVI_traverse(_handle->ivi, sfh_tune_simulation, (void *)_handle); - if(_handle->sim_tuned_rs_cnt>EXPECT_SIGNATURE_LEN) - { - _handle->blocksize*= MULTIPLE; - IVI_traverse(_handle->ivi, sfh_tune_callback, (void *)_handle); - } - else - { - break; - } - - }while(_handle->s_state_cnt>EXPECT_SIGNATURE_LEN); - return; -} -unsigned int SFH_feed(sfh_instance_t * handle, const char * data, unsigned int size, unsigned long long offset) -{ - fuzzy_handle_inner_t * _handle=(fuzzy_handle_inner_t *)handle; - if(data == NULL || size == 0) - { - return 0; - } - unsigned int length = segment_overlap(_handle, size, offset, data); - _handle->effective_length += length; - _handle->length_increase += length; - if(_handle->s_state_cnt>EXPECT_SIGNATURE_LEN&&_handle->do_tune==1) - { - unsigned long long check_length = (_handle->effective_length/_handle->s_state_cnt)*EXPECT_SIGNATURE_LEN; - - if(_handle->length_increase > check_length) - { - do_sfh_tune(handle); - _handle->length_increase = 0; - } - } -#if 0 - SFH_digest(handle,result, sizeof(result)); - printf("%llu %s\n",offset,result); -#endif - return length; -} - - - - - -unsigned long long get_blocksize(unsigned long long orilen) -{ - double tmp = orilen/(64 * BLOCKSIZE_MIN); - double index = floor(log(tmp)/log(2)); - double tmp_t = pow(2, index); - unsigned long long blocksize = (unsigned long long)(tmp_t * BLOCKSIZE_MIN); - if(blocksize == 0) - { - blocksize = BLOCKSIZE_MIN; - } - return blocksize; -// return BLOCKSIZE_MIN; -} - -sfh_seg_t* create_sfh_seg(fuzzy_handle_inner_t * _handle,unsigned long long offset) -{ - sfh_seg_t*p=(sfh_seg_t*)calloc(sizeof(sfh_seg_t),1); - roll_init(&(p->r_state)); - p->s_size = INIT_SIZE; - p->s_cnt=0; - p->r_size = INIT_SIZE; - p->r_cnt=0; - p->left_offset=p->right_offset=offset; - p->r_array = (unsigned int*)malloc(sizeof(unsigned int)*(p->r_size)); - _handle->fuzzy_node_memory+=sizeof(unsigned int)*(p->r_size); - p->s_array = (struct zt_state_t*)malloc(sizeof(struct zt_state_t)*(p->s_size)); - _handle->fuzzy_node_memory+=sizeof(struct zt_state_t)*(p->s_size); - zt_hash_initial(&(p->s_state)); - zt_hash_initial(&(p->p_state)); - _handle->fuzzy_node_memory += sizeof(sfh_seg_t); - return p; -} -//return freed memory size -int destroy_sfh_seg(sfh_seg_t*p) -{ - int ret_size=0; - if(p->s_array != NULL) - { - free(p->s_array); - p->s_array=NULL; - ret_size+=p->s_size*sizeof(struct zt_state_t); - } - if(p->r_array != NULL) - { - free(p->r_array); - p->r_array=NULL; - ret_size+=p->r_size*sizeof(unsigned int); - } - ret_size+=sizeof(sfh_seg_t); - free(p); - p=NULL; - return ret_size; -} -/** - * ÅжÏÊý¾ÝÊÇ·ñÓëÒѾ­¼ÆËã¹ýµÄÊý¾ÝÓи²¸Ç - */ -unsigned int segment_overlap(fuzzy_handle_inner_t * _handle, unsigned int size, unsigned long long offset, const char * data) -{ - IVI_seg_t ** overlap_segs = NULL; - IVI_seg_t *new_seg=NULL,*target_seg=NULL; - sfh_seg_t* sfh_seg=NULL; - int overlap_segnum = 0,i=0,co_seg_num=0,ret=0; - unsigned int effective_length = 0; - unsigned long long calc_begin=offset; - unsigned long long calc_end=offset+size-1; - - //printf("size: %u\n",size); - //printf("before query\n"); - /*²éѯÊÇ·ñÓи²¸Ç£¬Èç¹ûÓи²¸Ç£¬·µ»Ø¸²¸ÇµÄsegmentµÄƬÊý£¬Èç¹ûûÓи²¸Ç£¬·µ»Ø0*/ - if(offset>0) - { - overlap_segnum = IVI_query(_handle->ivi, offset-1, offset + size, &overlap_segs); - } - else - { - overlap_segnum = IVI_query(_handle->ivi, 0, offset + size, &overlap_segs); - } - IVI_seg_t * co_overlap_segs[overlap_segnum+1]; - assert(overlap_segnum>=0); - - if(overlap_segnum==0||offsetleft) - { - sfh_seg=create_sfh_seg(_handle,offset); - calc_begin=offset; - if(overlap_segnum == 0) - { - calc_end=offset+size-1; - } - else - { - calc_end=MIN(overlap_segs[0]->left-1,offset+size-1); - } - new_seg = IVI_seg_malloc(calc_begin, calc_end, (void *)sfh_seg); - _handle->s_state_cnt+=sfh_update_seg(_handle, sfh_seg,data+calc_begin-offset, calc_end-calc_begin+1, _handle->blocksize); - effective_length+=(calc_end-calc_begin+1); - co_overlap_segs[co_seg_num]=new_seg; - co_seg_num++; - } - for(i=0;iivi,overlap_segs[i]); - _handle->IVI_memory = IVI_mem_occupy(_handle->ivi); - assert(ret==0); - } - for(i=0;iright+1,calc_begin); - if(i+1left-1,offset+size-1); - } - else - { - calc_end=offset+size-1; - } - if(!after(calc_begin,calc_end)) - { - sfh_seg=(sfh_seg_t*)(co_overlap_segs[i]->data); - _handle->s_state_cnt+=sfh_update_seg(_handle,sfh_seg,data+calc_begin-offset, calc_end-calc_begin+1, _handle->blocksize); - effective_length+=(calc_end-calc_begin+1); - co_overlap_segs[i]->right+=calc_end-calc_begin+1; - calc_begin=calc_end+1; - } - } - target_seg=co_overlap_segs[0]; - for(i=0;idata)->r_index>0&&((sfh_seg_t*)co_overlap_segs[i]->data)->r_index>0) - { - memset(&result_p,0,sizeof(result_p)); - result_p.data=rp_buff; - result_p.size=sizeof(rp_buff); - sfh_output_callback(target_seg,&result_p); - memset(&result_n,0,sizeof(result_n)); - result_n.data=rn_buff; - result_n.size=sizeof(rn_buff); - sfh_output_callback(co_overlap_segs[i],&result_n); - printf("%s[%llu:%llu] %s[%llu:%llu]\n",rp_buff,target_seg->left, - target_seg->right, - rn_buff,co_overlap_segs[i]->left, - co_overlap_segs[i]->right); - } -#endif - _handle->s_state_cnt+=sfh_merge_seg(_handle,(sfh_seg_t*)target_seg->data, (sfh_seg_t*)co_overlap_segs[i]->data, _handle->blocksize); - target_seg->right=co_overlap_segs[i]->right; - IVI_seg_free(co_overlap_segs[i], fuzzy_node_free, (void *)_handle); - } - //IVI_seg_t * insert_seg=NULL; - //insert_seg = IVI_seg_malloc(target_seg->left, target_seg->right, target_seg->data); - ret=IVI_insert(_handle->ivi,target_seg); - _handle->IVI_memory = IVI_mem_occupy(_handle->ivi); - assert(ret==0); - free(overlap_segs); - return effective_length; -} - -int cmp(const void * a, const void * b) -{ - unsigned int tmp_a = *(unsigned int *)a; - unsigned int tmp_b = *(unsigned int *)b; - if(before(tmp_a, tmp_b)) - { - return -1; - } - else if(after(tmp_a, tmp_b)) - { - return 1; - } - else - { - return 0; - } -} - -double get_rs_entropy(unsigned int * r_array, unsigned int r_index) -{ - qsort(r_array, r_index, sizeof(unsigned int), cmp); - unsigned int current_r = r_array[0]; - unsigned int * tmp_r = r_array; - unsigned int count = 0; - double sum = 0; - int i = 0; - for(i = 0; i <= r_index; i++) - { - if(i == r_index || *tmp_r != current_r) - { - double p = (double)count/r_index; - //printf("count : %d\n",count); - //printf("r_index: %u\n",r_index); - //printf("p:%f\n",p); - if(p != 0) - { - sum += p * (log(p)/log(2)); - } - current_r = *tmp_r; - count = 0; - } - else - { - count++; - } - if(i < r_index) - { - tmp_r ++; - } - } - return (-sum); - -} - - - - -int write_uint_array(unsigned int ** array,unsigned int *index, unsigned int *size,unsigned int value) -{ - int mem_size=0; - if(*index==*size) - { - (*size)*=2; - mem_size+=*size; - *array=(unsigned int*)realloc(*array,sizeof(unsigned int)*(*size)); - } - (*array)[*index]=value; - (*index)++; - return mem_size; -} - -int sfh_update_seg(fuzzy_handle_inner_t * _handle, sfh_seg_t * p, const char * data, unsigned long data_size,unsigned long long blocksize) -{ - unsigned long i = 0; - unsigned int roll_hash_value = 0; - int state_inc_cnt=0; - if(p->msize < ROLLING_WINDOW - 1) - { - for(i = 0; i < ROLLING_WINDOW - p->msize && i < data_size; i++) - { - p->mbuf[p->msize + i] = data[i]; - roll_hash(&(p->r_state), data[i]); - } - p->msize += i; - } - for(; i < data_size; i++) - { - roll_hash(&(p->r_state), data[i]); - roll_hash_value = roll_sum(&(p->r_state)); - - zt_hash(&(p->s_state),data[i]); - if((roll_hash_value % (blocksize)) == blocksize - 1) - { - p->slice_num ++; - if(p->r_cnt==0) - { - p->p_state.val=p->s_state.val; - } - else - { -#ifdef DEBUG_PRINT - printf("p->s_cnt:%u\n",p->s_cnt); - printf("p->s_size:%u\n",p->s_size); -#endif - _handle->fuzzy_node_memory+=write_uint_array((unsigned int**)(&(p->s_array)), &(p->s_cnt),&(p->s_size),p->s_state.val); - state_inc_cnt++; - } -#ifdef DEBUG_PRINT - printf("p->r_cnt:%u\n",p->s_cnt); - printf("p->r_size:%u\n",p->s_size); -#endif - _handle->fuzzy_node_memory+=write_uint_array(&(p->r_array),&(p->r_cnt),&(p->r_size),roll_hash_value); - zt_hash_initial(&(p->s_state)); - } - } - assert(p->r_cnt>=p->s_cnt); - p->right_offset+=data_size; - return state_inc_cnt; -} - -int sfh_merge_seg(fuzzy_handle_inner_t * _handle, sfh_seg_t * p, sfh_seg_t * n,unsigned long long blocksize) -{ - - unsigned int roll_hash_value = 0; - int i = 0,state_inc_cnt=0; - struct roll_state_t * rs = &(p->r_state); - for(i = 0; i < n->msize; i++) - { - roll_hash(rs, n->mbuf[i]); - roll_hash_value = roll_sum(rs); - zt_hash(&(p->s_state), n->mbuf[i]); - if(roll_hash_value % blocksize == blocksize - 1) - { - p->slice_num ++; - if(p->r_cnt == 0) - { - p->p_state.val = p->s_state.val; - } - else - { - _handle->fuzzy_node_memory+=write_uint_array((unsigned int **)(&(p->s_array)), &(p->s_cnt), &(p->s_size), p->s_state.val); - state_inc_cnt++; - } - _handle->fuzzy_node_memory+=write_uint_array(&(p->r_array),&(p->r_cnt), &(p->r_size), roll_hash_value); - zt_hash_initial(&(p->s_state)); - } - } - if(n->r_cnt==0) - { - zt_hash_arymul(&(p->s_state),&(n->p_state)); - zt_hash_arymul(&(p->s_state), &(n->s_state)); - } - else - { - if(p->r_cnt==0) - { - zt_hash_arymul(&(p->s_state),&(n->p_state)); - p->p_state.val=p->s_state.val; - } - else - { - zt_hash_arymul(&(p->s_state), &(n->p_state)); - _handle->fuzzy_node_memory+=write_uint_array((unsigned int **)(&(p->s_array)), &(p->s_cnt), &(p->s_size), p->s_state.val); - state_inc_cnt++; - } - p->s_state.val=n->s_state.val; - } - for(i=0;ir_cnt;i++) - { - _handle->fuzzy_node_memory+=write_uint_array(&(p->r_array),&(p->r_cnt), &(p->r_size), n->r_array[i]); - } - for(i=0;is_cnt;i++) - { - _handle->fuzzy_node_memory+=write_uint_array((unsigned int **)(&(p->s_array)), &(p->s_cnt), &(p->s_size), n->s_array[i].val); - } - memcpy(&(p->r_state),&(n->r_state),sizeof(p->r_state)); - assert(p->r_cnt>=p->s_cnt); - p->right_offset=n->right_offset; - return state_inc_cnt; -} - -/** - * È¡³öÇø¼äÁ´±íÀïÃæµÄhash_resultÖµ£¬²¢½øÐÐÆ´½Ó£¬ÐγÉ×îºóµÄresultÊä³ö£¬²¢ÇÒÂú×ãabc[1:100]def[200:300]ÕâÖÖ¸ñʽ - */ -int SFH_digest(sfh_instance_t * handle, char * hash_buffer, unsigned int size) -{ - fuzzy_handle_inner_t* _handle=(fuzzy_handle_inner_t *)handle; - unsigned int estimate_len=_handle->s_state_cnt+IVI_seg_cnt(_handle->ivi)*24+1; - int actual_len=0; - char* p=NULL; - sfh_output_t result; - memset(&result,0,sizeof(result)); - result.size_b1 = estimate_len; - result.size_b2 = estimate_len; - result.hash_b1 = (char*)calloc(sizeof(char),estimate_len); - result.hash_b2 = (char*)calloc(sizeof(char),estimate_len); - result.offset_b1 = 0; - result.offset_b2 = 0; - result.b1=_handle->blocksize; - result.b2=_handle->blocksize*MULTIPLE; - - IVI_traverse(_handle->ivi, sfh_output_callback, (void *) &result); - - if(result.offset_b1==0||result.offset_b2==0) - { - hash_buffer[0]='\0'; - goto fast_out; - } - if(result.last_char_b1!='\0') - { - p =strrchr(result.hash_b1,'['); - assert(p!=NULL); - memmove(p+1,p,strlen(p)); - *p=result.last_char_b1; - } - if(result.last_char_b2!='\0') - { - p =strrchr(result.hash_b2,'['); - assert(p!=NULL); - memmove(p+1,p,strlen(p)); - *p=result.last_char_b2; - } - actual_len=snprintf(hash_buffer,size,"%llu:%s#%llu:%s",result.b1,result.hash_b1, - result.b2,result.hash_b2); -fast_out: - free(result.hash_b1); - result.hash_b1=NULL; - free(result.hash_b2); - result.hash_b2=NULL; - return actual_len; -} -sfh_seg_t* sfh_clone_seg(sfh_seg_t* origin) -{ - sfh_seg_t* clone=NULL; - clone=(sfh_seg_t*)calloc(sizeof(sfh_seg_t),1); - memcpy(clone,origin,sizeof(sfh_seg_t)); - clone->s_array=calloc(sizeof(struct zt_state_t),clone->s_size); - memcpy(clone->s_array,origin->s_array,sizeof(struct zt_state_t)*clone->s_size); - clone->r_array=calloc(sizeof(unsigned int),clone->r_size); - memcpy(clone->r_array,origin->r_array,sizeof(unsigned int)*clone->r_size); - return clone; -} -int sfh_print_seg(sfh_seg_t* p, char* hash_result, int size,char* last_char) -{ - int idx=0,i=0; - if(p->left_offset== 0) - { - hash_result[idx] = map_to64bytes[zt_hash_code(&(p->p_state)) & 0x3F]; - idx++; - } - for(i = 0; i < p->s_cnt&&idxs_array[i].val) & 0x3F]; - } - if(p->s_state.val!=*((unsigned int*)ZT_INIT_VAL)) - { - *last_char=map_to64bytes[zt_hash_code(&(p->s_state)) & 0x3F]; - } - else - { - *last_char='\0'; - } - // p->right_offset-1 to get a closed interval - idx+=snprintf(hash_result+idx,size-idx,"[%llu:%llu]",p->left_offset, p->right_offset-1); - assert(idxdata); - sfh_seg_t* tmp; - if(node->s_cnt==0&&!(seg->left==0&&node->s_cnt > 0)) - { - return; - } - result->offset_b1+=sfh_print_seg(node,result->hash_b1+result->offset_b1,result->size_b1-result->offset_b1,&(result->last_char_b1)); - tmp=sfh_clone_seg(node); - sfh_tune_seg(tmp, result->b2); - result->offset_b2+=sfh_print_seg(tmp,result->hash_b2+result->offset_b2,result->size_b2-result->offset_b2,&(result->last_char_b2)); - destroy_sfh_seg(tmp); - tmp=NULL; - return; -} - -/** - * ¼ÆËãfuzzy_hashµÄ¸÷ÖÖ³¤¶È - */ -unsigned long long SFH_status(sfh_instance_t * handle, int type) -{ - unsigned long long length; - fuzzy_handle_inner_t * _handle = (fuzzy_handle_inner_t *)(handle); - final_length tmp_length; - char buffer[64]; - switch(type) - { - case TOTAL_LENGTH: //ÒѾ­¼ÆËã¹ýhashÖµµÄÈ«²¿³¤¶È - length = IVI_seg_length(_handle->ivi); - break; - case EFFECTIVE_LENGTH: //°üº¬ÔÚ¼ÆËãhashÖµÀïÃæµÄÓÐЧ³¤¶È - length = _handle->effective_length; - break; - case HASH_LENGTH: //×îºóÊä³ö¹þÏ£½á¹ûµÄ³¤¶È - tmp_length.hash_length = 0; - tmp_length.first_ZTH_offset = 0; - tmp_length.last_ZTH_offset = 0; - tmp_length.hash_length+=snprintf(buffer,sizeof(buffer),"%llu:",_handle->blocksize); - IVI_traverse(_handle->ivi, fuzzy_hash_length, (void *)&tmp_length); - length = tmp_length.hash_length + 1; - break; - case MEMORY_OCCUPY: - length = _handle->fuzzy_node_memory + _handle->IVI_memory; - break; - default: - return 0; - } - return length; -} - -void fuzzy_hash_length(IVI_seg_t * seg, void * user_para) -{ - char buffer[100]; - final_length * tmp = (final_length *)user_para; - sfh_seg_t * node = (sfh_seg_t *)(seg->data); - if(node->s_cnt==0&&!(seg->left==0&&node->r_cnt > 0)) - { - return; - } - snprintf(buffer, sizeof(buffer), "[%llu:%llu]", seg->left, seg->right); - tmp->hash_length += 2*node->r_cnt*sizeof(char) + 2*strlen(buffer); - return; -} - - diff --git a/src/inc_internal/FQDN_engine.h b/src/inc_internal/FQDN_engine.h deleted file mode 100644 index d2213fc..0000000 --- a/src/inc_internal/FQDN_engine.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * - * Copyright (c) 2020 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2020-09-01 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef H_FQDN_ENGINE_H -#define H_FQDN_ENGINE_H - -#ifdef __cplusplus -extern "C" { -#endif - - #include - - struct FQDN_rule - { - unsigned int id; - int is_suffix_match; /* is_suffix_match==0: exact match; is_suffix_match==1: longest suffix matching. */ - size_t len; - char * FQDN; /* Non-ASCII character is allowed. */ - void * user_tag; /* A transparent user tag for convenient accessing, the caller is responsible for its memory management. */ - }; - - struct FQDN_engine; - - struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule); - - struct FQDN_match - { - unsigned int id; - unsigned int offset; /* offset==0 for exact matching; offset>0 for longest suffix matching. */ - void * user_tag; - }; - - /* - *Function: - * Search FQDN in the rule base - *Paramters: - * instance[in]: Instance of FQDN engine - * FQDN[in]: FQDN for search - * FQDN_len[in]: Length of FQDN - * results[out]: An array to store matched FQDNs - * n_result[in]: Number of element in the result array - * Return: - * 0: No matched FQDN; - * >0: Number of matched FQNDs which were stored in results; - * <0: Error. - */ - int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); - - void FQDN_engine_free(struct FQDN_engine * instance); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h deleted file mode 100644 index a3369e0..0000000 --- a/src/inc_internal/Maat_ex_data.h +++ /dev/null @@ -1,30 +0,0 @@ - -#include "Maat_rule.h" -#include "Maat_garbage_collection.h" -struct EX_data_rt; - -struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); -void EX_data_rt_free(struct EX_data_rt* p); -void EX_data_rt_update_commit(struct EX_data_rt* ex_rt); - -void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema); -void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row); - -const char* EX_data_rt_cached_row_get(struct EX_data_rt* p, size_t index); - -void EX_data_rt_clear_row_cache(struct EX_data_rt* p); -size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* p); - -int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, - const char* row, const char* key, size_t key_len, - void* user_data, void* logger); - - -int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, void *logger); -MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len); -MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container); -size_t EX_data_rt_list_updating_ex_containers(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array); -void* EX_data_container_get_user_data(struct EX_data_container* ex_container); -size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt); - - diff --git a/src/inc_internal/Maat_garbage_collection.h b/src/inc_internal/Maat_garbage_collection.h deleted file mode 100644 index 2c2e1da..0000000 --- a/src/inc_internal/Maat_garbage_collection.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - -struct Maat_garbage_bin{}; -struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout); -void Maat_garbage_bin_free(struct Maat_garbage_bin* bin); -void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *)); -void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin); -size_t Maat_garbage_bin_get_size(struct Maat_garbage_bin* bin); -void Maat_garbage_collect_by_force(struct Maat_garbage_bin* bin); - diff --git a/src/inc_internal/Maat_hierarchy.h b/src/inc_internal/Maat_hierarchy.h deleted file mode 100644 index dc65f93..0000000 --- a/src/inc_internal/Maat_hierarchy.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "Maat_rule.h" -#include "Maat_garbage_collection.h" -#include - -struct Maat_hierarchy; -struct Maat_hierarchy* Maat_hierarchy_new(int thread_num, void* mesa_handle_logger, struct Maat_garbage_bin* bin); -void Maat_hierarchy_free(struct Maat_hierarchy* hier); -void Maat_hierarchy_set_compile_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)); -void Maat_hierarchy_set_region_user_data_free_func(struct Maat_hierarchy* hier, void (* func)(void *)); - - - -struct Maat_hierarchy_compile_mid; -struct Maat_hierarchy_compile_mid* Maat_hierarchy_compile_mid_new(struct Maat_hierarchy* hier, int thread_num); -void Maat_hierarchy_compile_mid_free(struct Maat_hierarchy_compile_mid* mid); -void Maat_hierarchy_compile_mid_udpate(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result); -void Maat_hierarchy_compile_user_data_iterate(struct Maat_hierarchy* hier, void (*callback)(void *user_data, void* apram), void* param); - -int Maat_hierarchy_compile_mid_has_NOT_clause(struct Maat_hierarchy_compile_mid* mid); - -size_t Maat_hierarchy_get_hit_paths(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, - struct Maat_hit_path_t* hit_paths, size_t n_path); - -int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int declared_clause_num, void* user_data); -int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id); - -void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id); -void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id); - -void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int region_id); - - -int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id); -int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id); -int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id); -int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int group_id, int superior_group_id); - -int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id, int region_id, int table_id, void* user_data); -int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int group_id, int region_id); - -int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier); -int Maat_hierarchy_region_compile(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile_mid* mid, int is_last_compile, void** user_data_array, size_t ud_array_sz); - - diff --git a/src/inc_internal/Maat_limits.h b/src/inc_internal/Maat_limits.h deleted file mode 100644 index 7fea843..0000000 --- a/src/inc_internal/Maat_limits.h +++ /dev/null @@ -1,10 +0,0 @@ -#define MAX_TABLE_LINE_SIZE (1024*16) -#define MAX_EXPR_KEYLEN 1024 -#define MAX_DISTRICT_LEN 64 - -#define MAX_SCANNER_HIT_NUM 4096 - -#define MAX_FAILED_NUM 128 - -#define MAX_MAAT_STAT_NUM 64 - diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h deleted file mode 100644 index 6340c6c..0000000 --- a/src/inc_internal/Maat_rule_internal.h +++ /dev/null @@ -1,416 +0,0 @@ -#pragma once - -#include "Maat_rule.h" -#include "Maat_command.h" -#include "Maat_limits.h" -#include "Maat_table_schema.h" -#include "Maat_table_runtime.h" - -#include -#include -#include -#include -#include "hiredis.h" -#include "IPMatcher.h" -#include "stream_fuzzy_hash.h" -#include "gram_index_engine.h" -#include "alignment_int64.h" -#include "FQDN_engine.h" -#include -#include -#include -#include - - - -#define mr_region_id_var "SEQUENCE_REGION" -#define mr_group_id_var "SEQUENCE_GROUP" - - - - - -typedef void* rule_scanner_t; - - - -struct db_expr_rule_t -{ - int region_id; - int group_id; - char keywords[MAX_EXPR_KEYLEN+1]; - char district[MAX_DISTRICT_LEN+1]; - enum MAAT_EXPR_TYPE expr_type; - enum MAAT_MATCH_METHOD match_method; - int is_hexbin; - int is_case_sensitive; - int is_valid; - -}; -struct db_ip_rule_t -{ - int region_id; - int group_id; - int addr_type; - union - { - //ip address use network order - //port use host order - ipv4_rule_t ipv4_rule; - ipv6_rule_t ipv6_rule; - }; - int is_valid; -}; -struct db_interval_rule -{ - int region_id; - int group_id; - interval_rule_t intval; - int is_valid; - char district[MAX_DISTRICT_LEN+1]; -}; -struct db_digest_rule -{ - int region_id; - int group_id; - unsigned long long orgin_len; - char* digest_string; - short confidence_degree; - int is_valid; -}; -struct db_fqdn_rule -{ - int region_id; - int group_id; - int is_suffix_match; - char* fqdn; - int is_valid; -}; - -struct Maat_rule_head -{ - int config_id; - int service_id; - char do_log; - char do_blacklist; - char action; - char resevered; - int serv_def_len; -}; - -#define COMPILE_RULE_MAGIC 0x1a2b3c4d -struct Maat_compile_rule -{ - long long magic_num; - struct Maat_rule_head head;// fix len of Maat_rule_t - char* service_defined; - int is_valid; - int declared_clause_num; - double evaluation_order; - const struct Maat_table_schema* ref_table; - MAAT_RULE_EX_DATA* ads; - int compile_id; - pthread_rwlock_t rwlock; -}; -struct db_group2group_rule -{ - int group_id; - int superior_group_id; - int is_valid; -}; -struct db_group2compile_rule -{ - int group_id; - int compile_id; - int is_valid; - int not_flag; - int virtual_table_id; - int clause_index; -}; -struct db_group_rule_t -{ - int group_id; - int parent_id; - int is_valid; - int not_flag; - int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. - int virtual_table_id; - int clause_id; -}; -#define RULESCAN_OP_ADD 0 -#define RULESCAN_OP_DEL 1 -struct op_expr_t -{ - boolean_expr_t* p_expr; - scan_rule_t* p_rules[MAAT_MAX_EXPR_ITEM_NUM]; - int convert_failed; - int no_effect_convert_cnt; - int table_id; - int rule_type; -}; -#define REGION_RULE_MAGIC 0x4d3c2b1a -struct Maat_region_inner -{ - long long magic_num; - int region_id; - int group_id; - int district_id; - int table_id; - int expr_id_cnt; - int expr_id_lb; //low boundary - int expr_id_ub; //up boundary -}; - -#define DISTRICT_ANY -1 -#define DISTRICT_UNKNOWN -2 -struct _OUTER_scan_status_t -{ - struct _Maat_feather_t* feather; - signed short thread_num; - unsigned char is_set_district; - unsigned char is_last_scan; - int district_id; //-1: Any District; -2: Unkonwn District; - int scan_cnt; - struct Maat_hierarchy_compile_mid* compile_mid; -}; - -struct iconv_handle_t -{ - int is_initialized; - iconv_t cd; -}; -struct _stream_para_t -{ - struct _Maat_feather_t* feather; - const struct Maat_table_schema* p_real_table; - int virtual_table_id; - int last_full_version; - struct Maat_scanner *ref_scanner; - - int thread_num; - int max_cross_size; - int caching_size; - char do_merge; - char do_expr:4; - char do_regex:4; - char* last_cache; - char* scan_buff; - void* rs_stream_para; - long process_offset; - unsigned long long total_len; - sfh_instance_t *fuzzy_hash_handle; - pthread_mutex_t fuzzy_mutex; - unsigned char query_point[8]; -}; - -struct rule_tag -{ - char* tag_name; - char* tag_val; -}; -struct Maat_scanner -{ - long long version; - time_t last_update_time; - mcore_long_t ref_cnt; - rule_scanner_t region; - size_t xx_plugin_rule_to_update_cnt; - struct Maat_table_runtime_manager* table_rt_mgr; - size_t max_table_num; - - struct Maat_hierarchy * hier; - struct Maat_garbage_bin* ref_garbage_bin; - - - struct maat_kv_store* district_map; - struct maat_kv_store* tmp_district_map; - - - int most_popular_sub_group; - unsigned long long max_presented_top_group_cnt; - - unsigned int district_num; - unsigned int cfg_num; - unsigned int exprid_generator; - unsigned int dedup_expr_num; - MESA_lqueue_head region_update_q; - - scan_result_t *region_rslt_buff; - GIE_result_t* gie_rslt_buff; - void* logger_ref; - - int max_thread_num; - iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] -}; -enum data_source -{ - SOURCE_NONE=0, - SOURCE_REDIS, - SOURCE_IRIS_FILE, - SOURCE_JSON_FILE -}; -struct source_iris_ctx -{ - char inc_dir[MAX_TABLE_NAME_LEN]; - char full_dir[MAX_TABLE_NAME_LEN]; -}; -struct source_json_ctx -{ - char json_file[MAX_TABLE_NAME_LEN]; - char iris_file[MAX_TABLE_NAME_LEN]; - char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; - struct timespec last_md5_time; -}; -struct source_redis_ctx -{ - redisContext *read_ctx; - redisContext *write_ctx; - char redis_ip[64]; - int redis_port; - int redis_db; - time_t last_reconnect_time; -}; -struct _Maat_feather_t -{ - struct Maat_scanner *scanner; - struct Maat_scanner *update_tmp_scanner; - struct Maat_garbage_bin * garbage_bin; - struct Maat_table_manager* table_mgr; - - int DEFERRED_LOAD_ON; - enum data_source input_mode; - union - { - struct source_iris_ctx iris_ctx; - struct source_json_ctx json_ctx; - struct source_redis_ctx mr_ctx; - }; - int is_running; - int rule_update_checking_interval_ms; - int rule_effect_interval_ms; - int garbage_collection_timeout_ms; - int cumulative_update_off; - int stat_on; - int perf_on; - int output_prometheus; - - void* logger; - long long maat_version; - long long last_full_version; - int scan_thread_num; - int rule_scan_type; - - char stat_file[MAX_TABLE_NAME_LEN]; - char instance_name[MAX_TABLE_NAME_LEN]; - char table_info_fn[MAX_TABLE_NAME_LEN]; - char compile_tn[MAX_TABLE_NAME_LEN]; - char group2compile_tn[MAX_TABLE_NAME_LEN]; - char group2group_tn[MAX_TABLE_NAME_LEN]; - char group_tn[MAX_TABLE_NAME_LEN]; - pthread_mutex_t background_update_mutex; - char decrypt_key[MAX_TABLE_NAME_LEN]; - char decrypt_algo[MAX_TABLE_NAME_LEN]; - int maat_json_is_gzipped; - pthread_t cfg_mon_t; - - int AUTO_NUMBERING_ON; - -// redisContext *redis_write_ctx; // not thread safe. - - int cmd_q_cnt; - struct _Maat_cmd_inner_t* cmd_qhead, *cmd_qtail; - long long base_rgn_seq,base_grp_seq,server_time; - long long load_version_from; - - char* accept_tags_raw; - struct rule_tag *accept_tags; - int n_tags; - - char foreign_cont_dir[MAX_TABLE_NAME_LEN]; -//internal states - long long new_version; - -//for scanner independent stat>>>> - int backgroud_update_enabled; - screen_stat_handle_t stat_handle; - int total_stat_id; - int fs_status_id[MAX_MAAT_STAT_NUM]; - int fs_column_id[MAX_MAAT_STAT_NUM]; - mcore_long_t outer_mid_cnt; - mcore_long_t compile_mid_cnt; - mcore_long_t hit_cnt; - mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, - mcore_long_t not_grp_hit_cnt; - long long total_scan_bytes; - long long total_scan_cnt; - long long update_err_cnt;//sum of the same name variable in each table - long long iconv_err_cnt;//sum of the same name variable in each table - long long scan_err_cnt; - long long zombie_rs_stream; - long long postpone_q_size; - long long compile_rule_num; - long long cmd_acc_num; - long long line_cmd_acc_num; -}; - -struct foreign_key -{ - int column; - char* key; - char* filename; -}; -struct serial_rule_t //rm= Redis Maat -{ - enum MAAT_OPERATION op;//0: delete, 1: add. - unsigned long rule_id; - int label_id; - long long timeout; // absolute unix time. - char table_name[256]; - char* table_line; - int n_foreign; - struct foreign_key* f_keys; - TAILQ_ENTRY(serial_rule_t) entries; -}; -int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); - -void maat_start_cb(long long new_version,int update_type,void*u_para); -int maat_update_cb(const char* table_name,const char* line,void *u_para); -void maat_finish_cb(void* u_para); -void *thread_rule_monitor(void *arg); -unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge); - -void * HASH_fetch_by_id(MESA_htable_handle hash,int id); -int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); -int HASH_delete_by_id(MESA_htable_handle hash,int id); -void maat_read_full_config(_Maat_feather_t* _feather); -void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_output(struct _Maat_feather_t* feather); - -redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); -int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct Maat_table_manager* table_mgr, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off); -int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process); -int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, const char* dir,void *logger); -void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger); -void rewrite_table_line_with_foreign(struct serial_rule_t*p); - -void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len); -MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); -void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); - - -void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op, unsigned long rule_id,int label_id,const char* table_name,const char* line, long long timeout); -void empty_serial_rules(struct serial_rule_t* rule); -int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); -long long redis_server_time(redisContext* ctx); -redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); - -int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz); -void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, - void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para - int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para - void (*finish)(void*),//u_para - void* u_para, - const char* dec_key, - _Maat_feather_t* feather); - - diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h deleted file mode 100644 index 4cbea03..0000000 --- a/src/inc_internal/Maat_table_runtime.h +++ /dev/null @@ -1,120 +0,0 @@ -#include "Maat_table_schema.h" -#include "Maat_ex_data.h" - -#include "IPMatcher.h" -#include "gram_index_engine.h" -#include "bool_matcher.h" -#include "FQDN_engine.h" -#include "alignment_int64.h" -#include - -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct fqdn_plugin_runtime -{ - struct FQDN_engine* fqdn_engine; - struct EX_data_rt* ex_data_rt; - int changed_flag; -}; -struct bool_plugin_runtime -{ - struct bool_matcher *matcher; - struct EX_data_rt* ex_data_rt; - int changed_flag; -}; -struct plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - long long acc_line_num; -}; - -struct ip_plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - struct ip_matcher* ip_matcher; - size_t mem_use_by_ip_matcher; - int changed_flag; -}; -struct expr_ng_runtime -{ - struct EX_data_rt* ex_data_rt; - hs_database_t *hs_pure_literal_db; - hs_database_t *hs_regex_db; - struct bool_matcher *logical_matcher; - int changed_flag; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group2compile_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin - struct bool_plugin_runtime bool_plugin; - struct plugin_runtime plugin; - struct ip_plugin_runtime ip_plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group2compile_runtime group2compile; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; - struct Maat_garbage_bin* ref_garbage_bin; -}; - - -struct Maat_table_runtime_manager; -struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin); -void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); -struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); -void Maat_table_runtime_perf_stat(struct Maat_table_runtime* p, int scan_len, struct timespec* start, struct timespec* end,int thread_num); -enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* table_rt); - -size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt); -const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row); - -int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); -void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); -void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); -int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); - -void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt); -MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key); - -void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); -int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt); - -void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt); -int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); - -void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt); -int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_ids[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); - - diff --git a/src/inc_internal/Maat_table_schema.h b/src/inc_internal/Maat_table_schema.h deleted file mode 100644 index 5238b8b..0000000 --- a/src/inc_internal/Maat_table_schema.h +++ /dev/null @@ -1,236 +0,0 @@ -#pragma once -#include -#include "Maat_rule.h" -#include "Maat_limits.h" - -#define MAX_COMPILE_EX_DATA_NUM 2 -#define MAX_FOREIGN_CLMN_NUM 8 -#define MAX_PLUGIN_PER_TABLE 32 -#define MAX_CHARSET_NUM __CHARSET_MAX -#define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_TABLE_NAME_LEN 256 - - -enum USER_REGION_ENCODE -{ - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; -enum MAAT_SCAN_TYPE -{ - SCAN_TYPE_INVALID=-1, - SCAN_TYPE_NONE=0, - SCAN_TYPE_PLUGIN, - SCAN_TYPE_IP_PLUGIN, - SCAN_TYPE_FQDN_PLUGIN, - SCAN_TYPE_BOOL_PLUGIN, - SCAN_TYPE_IP, - SCAN_TYPE_INTERVAL, - SCAN_TYPE_STRING, - __SCAN_TYPE_MAX -}; - -enum MAAT_TABLE_COMPONENT_TYPE -{ - COMPONENT_TABLE_TYPE_NONE=-1, - COMPONENT_TABLE_TYPE_SOURCE_IP=0, - COMPONENT_TABLE_TYPE_DESTINATION_IP, - COMPONENT_TABLE_TYPE_SESSION, - __COMPONENT_TABLE_TYPE_MAX -}; - - -enum MAAT_TABLE_TYPE -{ - TABLE_TYPE_INVALID=-1, - TABLE_TYPE_EXPR=0, - TABLE_TYPE_IP, - TABLE_TYPE_IP_PLUS, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_DIGEST, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_INTERVAL_PLUS, - //Above are physical table for scan - TABLE_TYPE_VIRTUAL, - TABLE_TYPE_COMPOSITION, - TABLE_TYPE_GROUP2GROUP, - TABLE_TYPE_GROUP2COMPILE, - TABLE_TYPE_GROUP, - TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN, - TABLE_TYPE_IP_PLUGIN, - TABLE_TYPE_FQDN_PLUGIN, - TABLE_TYPE_BOOL_PLUGIN -}; - -struct compile_ex_data_idx -{ - Maat_rule_EX_new_func_t *new_func; - Maat_rule_EX_free_func_t* free_func; - Maat_rule_EX_dup_func_t* dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; -struct compile_table_schema -{ - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -}; - -struct string_table_schema -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch;//obsolete since 20190401 - long long iconv_err_cnt; -}; -struct virtual_table_schema -{ - int physical_table_id[__SCAN_TYPE_MAX]; -}; -struct composition_table_schema -{ - int component_table_id[__COMPONENT_TABLE_TYPE_MAX]; -}; -struct plugin_table_callback_schema -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; -struct EX_data_schema -{ - Maat_plugin_EX_new_func_t* new_func; - Maat_plugin_EX_free_func_t* free_func; - Maat_plugin_EX_dup_func_t* dup_func; - Maat_plugin_EX_key2index_func_t* key2index_func; - long argl; - void *argp; - int set_flag; -}; -struct plugin_table_schema -{ - int key_column; - int valid_flag_column; - int rule_tag_column; - int n_foreign; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - int cb_plug_cnt; - struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; - struct EX_data_schema ex_schema; -}; -struct ip_plugin_table_schema -{ - int row_id_column; - int ip_type_column; - int start_ip_column; - int end_ip_column; - int valid_flag_column; - int rule_tag_column; - int have_exdata; - struct EX_data_schema ex_schema; -}; -struct fqdn_plugin_table_schema -{ - - int row_id_column; - int is_suffix_flag_column; - int fqdn_column; - int valid_flag_column; - int rule_tag_column; - int have_exdata; - struct EX_data_schema ex_schema; -}; -struct bool_plugin_table_schema -{ - int row_id_column; - int bool_expr_column; - int rule_tag_column; - int valid_flag_column; - int have_exdata; - struct EX_data_schema ex_schema; - -}; -struct Maat_table_schema -{ - int table_id; - int conj_cnt; - int updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - union - { - struct compile_table_schema compile; - struct string_table_schema expr; - struct plugin_table_schema plugin; - struct ip_plugin_table_schema ip_plugin; - struct fqdn_plugin_table_schema fqdn_plugin; - struct bool_plugin_table_schema bool_plugin; - struct virtual_table_schema virtual_table; - struct composition_table_schema composition; - void* others;//group, ip, interval and digest don't have any special schema. - }; -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - int stat_line_id; -}; -struct Maat_table_manager; -struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger); -void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); -size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); -size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); - -struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id); -struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); -enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type); - -int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); -int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr, - int table_id, - Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para - Maat_update_callback_t *update,//table line ,u_para - Maat_finish_callback_t *finish,//u_para - void* u_para); -int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); - -const char* Maat_table_manager_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); -enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); - -int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, - Maat_rule_EX_new_func_t *new_func, - Maat_rule_EX_free_func_t* free_func, - Maat_rule_EX_dup_func_t* dup_func, - long argl, void *argp); -struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); -int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger); -struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema); - - -void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); -void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); - -int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); -struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); -void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name); - -int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type); -int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table); - diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h deleted file mode 100644 index 3cd5030..0000000 --- a/src/inc_internal/Maat_utils.h +++ /dev/null @@ -1,100 +0,0 @@ -#pragma once -#include "Maat_rule.h" -#include -#include -#include -#include -#include //fstat -#include //fstat -#include //fstat - -#define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) -#define FREE(p) {free(*p);*p=NULL;} - -#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) -#define atomic_inc(x) __sync_add_and_fetch((x),1) -#define atomic_dec(x) __sync_sub_and_fetch((x),1) -#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) -#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) - typedef int atomic_t; -#define ATOMIC_INIT(i) { (i) } -#define atomic_read(x) __sync_add_and_fetch((x),0) -#define atomic_set(x,y) __sync_lock_test_and_set((x),y) -#else -#include -#endif -#define TRUE 1 -#define FALSE 0 - -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif - -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) -#endif - -#define UNUSED __attribute__((unused)) -const char* module_name_str(const char*name); -#define maat_module (module_name_str("MAAT_Frame")) -#define TO_RELATION_ID(vid, gid) ((unsigned long long)vid<<32|gid) -char* _maat_strdup(const char* s); -char* str_unescape(char* s); -inline void ipv6_ntoh(unsigned int *v6_addr) -{ - unsigned int i=0; - for(i=0;i<4;i++) - { - v6_addr[i]=ntohl(v6_addr[i]); - } - return; -} -int hex2bin(char *hex,int hex_len,char *binary,int size); -char* str_tolower(char* string); -char *strtok_r_esc(char *s, const char delim, char **save_ptr); -char *str_unescape_and(char*s); -char* str_unescape(char* s); -size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src); - -pid_t gettid(void); -int system_cmd_mkdir(const char* path); -int system_cmd_rm(const char* src_file); -int system_cmd_mv(const char* src_file, const char*dst_file); -int system_cmd_cp(const char* src_file, const char*dst_file); -int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); -int system_cmd_gzip(const char* src_file, const char* dst_file); - -char* md5_file(const char* filename, char* md5string); -int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); -const char** charset_get_all_name(void); -const char* charset_get_name(enum MAAT_CHARSET charset); -int lqueue_destroy_cb(void *data, long data_len, void *arg); - -//Caller is responsible to free the out_uncompressed_data buffer. -int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz); - -int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); -int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); -//do_encrypt: 1 for encryption, 0 for decryption. -int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz); -enum MAAT_IP_FORMAT -{ - FORMAT_RANGE, - FORMAT_MASK, - FORMAT_CIDR, - FORMAT_UNKNOWN -}; -enum MAAT_IP_FORMAT ip_format_str2int(const char* format); -int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, const char* ip2, unsigned int range_begin[], unsigned int range_end[]); - - diff --git a/src/inc_internal/UniversalBoolMatch.h b/src/inc_internal/UniversalBoolMatch.h deleted file mode 100644 index 95cd6c5..0000000 --- a/src/inc_internal/UniversalBoolMatch.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * - * Copyright (c) 2014 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2014-12-09 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef H_UNIVERSAL_BOOL_MATCH_H -#define H_UNIVERSAL_BOOL_MATCH_H - -#ifdef __cplusplus -extern "C" -{ -#endif - - #define MAX_ITEMS_PER_BOOL_EXPR 8 - - typedef struct _universal_bool_expr_t - { - void * bool_expr_id; - unsigned int bool_item_num; - unsigned int bool_item_ids[MAX_ITEMS_PER_BOOL_EXPR]; - }universal_bool_expr_t; - - /*×¢Ò⣺±¾º¯Êýµ÷Óûύ»»bool_exprsÖÐÔªËصÄλÖÃ*/ - void * boolexpr_initialize(universal_bool_expr_t * bool_exprs, unsigned int bool_expr_num, unsigned int max_thread_num, unsigned int * mem_size); - - int boolexpr_match(void * instance, unsigned int thread_id, unsigned int * item_ids, unsigned int item_num, void ** result, unsigned int size); - - void boolexpr_destroy(void * instance); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc_internal/alignment_int64.h b/src/inc_internal/alignment_int64.h deleted file mode 100644 index a7a7d2a..0000000 --- a/src/inc_internal/alignment_int64.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef H_ALIGNMENT_INT64_H_INCLUDE -#define H_ALIGNMENT_INT64_H_INCLUDE - -#include - -#define CPU_CACHE_ALIGMENT 64 -typedef long long* mcore_long_t; - -inline mcore_long_t alignment_int64_array_alloc(int size) -{ - long long *ret=NULL; - ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size); - return ret; -} -inline long long alignment_int64_array_sum(mcore_long_t array,int size) -{ - long long sum=0; - int offset=0,i=0; - for(i=0;i0) - { - cnt++; - } - } - return cnt; -} -inline void alignment_int64_array_reset(mcore_long_t array,int size) -{ - memset(array, 0, CPU_CACHE_ALIGMENT*size); - return; -} -inline void alignment_int64_array_free(mcore_long_t array) -{ - free(array); -} - -#endif - diff --git a/src/inc_internal/cJSON.h b/src/inc_internal/cJSON.h deleted file mode 100644 index 6e0bde9..0000000 --- a/src/inc_internal/cJSON.h +++ /dev/null @@ -1,277 +0,0 @@ -/* - Copyright (c) 2009-2017 Dave Gamble and cJSON contributors - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. -*/ - -#ifndef cJSON__h -#define cJSON__h - -#ifdef __cplusplus -extern "C" -{ -#endif - -/* project version */ -#define CJSON_VERSION_MAJOR 1 -#define CJSON_VERSION_MINOR 7 -#define CJSON_VERSION_PATCH 7 - -#include - -/* cJSON Types: */ -#define cJSON_Invalid (0) -#define cJSON_False (1 << 0) -#define cJSON_True (1 << 1) -#define cJSON_NULL (1 << 2) -#define cJSON_Number (1 << 3) -#define cJSON_String (1 << 4) -#define cJSON_Array (1 << 5) -#define cJSON_Object (1 << 6) -#define cJSON_Raw (1 << 7) /* raw json */ - -#define cJSON_IsReference 256 -#define cJSON_StringIsConst 512 - -/* The cJSON structure: */ -typedef struct cJSON -{ - /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ - struct cJSON *next; - struct cJSON *prev; - /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ - struct cJSON *child; - - /* The type of the item, as above. */ - int type; - - /* The item's string, if type==cJSON_String and type == cJSON_Raw */ - char *valuestring; - /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ - int valueint; - /* The item's number, if type==cJSON_Number */ - double valuedouble; - - /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ - char *string; -} cJSON; - -typedef struct cJSON_Hooks -{ - void *(*malloc_fn)(size_t sz); - void (*free_fn)(void *ptr); -} cJSON_Hooks; - -typedef int cJSON_bool; - -#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) -#define __WINDOWS__ -#endif -#ifdef __WINDOWS__ - -/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 2 define options: - -CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols -CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) -CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol - -For *nix builds that support visibility attribute, you can define similar behavior by - -setting default visibility to hidden by adding --fvisibility=hidden (for gcc) -or --xldscope=hidden (for sun cc) -to CFLAGS - -then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does - -*/ - -/* export symbols by default, this is necessary for copy pasting the C and header file */ -#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_EXPORT_SYMBOLS -#endif - -#if defined(CJSON_HIDE_SYMBOLS) -#define CJSON_PUBLIC(type) type __stdcall -#elif defined(CJSON_EXPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllexport) type __stdcall -#elif defined(CJSON_IMPORT_SYMBOLS) -#define CJSON_PUBLIC(type) __declspec(dllimport) type __stdcall -#endif -#else /* !WIN32 */ -#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) -#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type -#else -#define CJSON_PUBLIC(type) type -#endif -#endif - -/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. - * This is to prevent stack overflows. */ -#ifndef CJSON_NESTING_LIMIT -#define CJSON_NESTING_LIMIT 1000 -#endif - -/* returns the version of cJSON as a string */ -CJSON_PUBLIC(const char*) cJSON_Version(void); - -/* Supply malloc, realloc and free functions to cJSON */ -CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); - -/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ -/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ -CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); -/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ -/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ -CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); - -/* Render a cJSON entity to text for transfer/storage. */ -CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); -/* Render a cJSON entity to text for transfer/storage without any formatting. */ -CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); -/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ -CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); -/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ -/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ -CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); -/* Delete a cJSON entity and all subentities. */ -CJSON_PUBLIC(void) cJSON_Delete(cJSON *c); - -/* Returns the number of items in an array (or object). */ -CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); -/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */ -CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); -/* Get item "string" from object. Case insensitive. */ -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); -CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); -/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ -CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); - -/* Check if the item is a string and return its valuestring */ -CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item); - -/* These functions check the type of an item */ -CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); -CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); - -/* These calls create a cJSON item of the appropriate type. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); -CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); -CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); -/* raw json */ -CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); -CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); -CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); - -/* Create a string where valuestring references a string so - * it will not be freed by cJSON_Delete */ -CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string); -/* Create an object/arrray that only references it's elements so - * they will not be freed by cJSON_Delete */ -CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child); -CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child); - -/* These utilities create an Array of count items. */ -CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); -CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count); - -/* Append item to the specified array/object. */ -CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); -/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. - * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before - * writing to `item->string` */ -CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); -/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ -CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); -CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); - -/* Remove/Detatch items from Arrays/Objects. */ -CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); -CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); - -/* Update array items. */ -CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ -CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); -CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); -CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); - -/* Duplicate a cJSON item */ -CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); -/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will -need to be released. With recurse!=0, it will duplicate any children connected to the item. -The item->next and ->prev pointers are always zero on return from Duplicate. */ -/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. - * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ -CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); - - -CJSON_PUBLIC(void) cJSON_Minify(char *json); - -/* Helper functions for creating and adding items to an object at the same time. - * They return the added item or NULL on failure. */ -CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name); -CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name); -CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name); -CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean); -CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number); -CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string); -CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw); -CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name); -CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name); - -/* When assigning an integer value, it needs to be propagated to valuedouble too. */ -#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) -/* helper for the cJSON_SetNumberValue macro */ -CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); -#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) - -/* Macro for iterating over an array or object */ -#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) - -/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ -CJSON_PUBLIC(void *) cJSON_malloc(size_t size); -CJSON_PUBLIC(void) cJSON_free(void *object); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc_internal/config_monitor.h b/src/inc_internal/config_monitor.h deleted file mode 100644 index 6167ce4..0000000 --- a/src/inc_internal/config_monitor.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef __CONFIG_MONITOR_H_INCLUDE_ -#define __CONFIG_MONITOR_H_INCLUDE_ - -#define CM_UPDATE_TYPE_FULL 1 -#define CM_UPDATE_TYPE_INC 2 - -void config_monitor_traverse(long long version,const char*idx_dir, - void (*start)(long long, int, void*),//vesion,CM_UPDATE_TYPE_*,u_para - int (*update)(const char*, const char*, void*),//table name ,line ,u_para - void (*finish)(void*),//u_para - void* u_para, - const char* dec_key, - void* logger); -#endif - diff --git a/src/inc_internal/dynamic_array.h b/src/inc_internal/dynamic_array.h deleted file mode 100644 index 11b3643..0000000 --- a/src/inc_internal/dynamic_array.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _DYNAMIC_ARRAY_H_INCLUDE_ -#define _DYNAMIC_ARRAY_H_INCLUDE_ -struct dynamic_array_t -{ - void ** array; - long long size; - long long enlarge_step; -}; -struct dynamic_array_t* dynamic_array_create(long long init_size, long long step); -void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)); -void* dynamic_array_read(struct dynamic_array_t* d_array,long long i); -void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data); -#endif //_DYNAMIC_ARRAY_H_INCLUDE_ diff --git a/src/inc_internal/hyperscan_adapter.h b/src/inc_internal/hyperscan_adapter.h deleted file mode 100644 index c2fef40..0000000 --- a/src/inc_internal/hyperscan_adapter.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once -#include -#ifdef __cplusplus -extern "C" -{ -#endif - -#define HSA_MAX_SUB_STRING_NUM 8 -struct hs_pattern -{ - char *pattern; - size_t pattern_len; - int start_offset; //-1: not specified; -2: match is start at the begining of the input - int end_offset; //-1: not specified; -2; match is end at the end of the input -}; - -struct hs_expression -{ - void * user_tag; - int is_regex; //if is_regex==1, then n_sub_pattern must be 1 - size_t n_sub_pattern; //must not exceed HSA_MAX_SUB_STRING_NUM - struct hs_pattern sub_patterns[]; -}; - -struct hs_adapter; -#define HSA_SCAN_MODE_BLOCK 0 -#define HSA_SCAN_MODE_STREAM 1 -/** - * @param scan_mode - * HSA_SCAN_MODE_BLOCK or HSA_SCAN_MODE_STREAM - * @param exprs - * Expressions to match, it's life time must be longger than the hs_adapter. -*/ -struct hs_adapter *hs_adapter_new(const struct hs_expression ** exprs, size_t n_expr, int scan_mode, int n_thread); -void hs_adapter_free(struct hs_adapter *adapter); - -int hs_adapter_scan(struct hs_adapter *adapter, int thread_id, const char* data, unsigned int length, void **matched_tags, size_t n_tag); - -#ifdef __cplusplus -} -#endif - diff --git a/src/inc_internal/interval_index.h b/src/inc_internal/interval_index.h deleted file mode 100644 index a67c7ea..0000000 --- a/src/inc_internal/interval_index.h +++ /dev/null @@ -1,313 +0,0 @@ -/************************************************************************ - * InterVal Index interface - * NOTE that: - * (1) There are no overlapping intervals in InterVal Index; - * (2) Each interval is closed; - * (3) The interval supports rollback. - * - * author: zhengchao@iie.ac.cn tangqi@iie.ac.cn - * last modify time: 2015-12-04 - *************************************************************************/ - -#ifndef _INTERVAL_INDEX_H_ -#define _INTERVAL_INDEX_H_ - -#ifdef __cplusplus -extern "C"{ -#endif - - -#define SIZE_8 - -#ifdef SIZE_8 -typedef unsigned long long OFFSET_TYPE; -typedef signed long long S_OFFSET_TYPE; -#else -typedef unsigned int OFFSET_TYPE; -typedef signed int S_OFFSET_TYPE; -#endif - - -typedef struct{ -}IVI_t; - - -/** - * structure of segment - **/ -typedef struct __IVI_seg_t{ - OFFSET_TYPE left; - OFFSET_TYPE right; - void * data; -}IVI_seg_t; - - -typedef void IVI_callback_t(IVI_seg_t * seg, void * usr_para); - -/** - * Deal with rollback - * Refering to the approach of Linux's kernel to solute tcp seq rollback - **/ -static inline int before(OFFSET_TYPE off1, OFFSET_TYPE off2) -{ - return (S_OFFSET_TYPE)(off1 - off2) < 0; -} -#define after(off2, off1) before(off1, off2) - -static inline int continuous(OFFSET_TYPE prev, OFFSET_TYPE next) -{ - return ((next - prev) == 1); -} - - -IVI_seg_t * IVI_first_seg(IVI_t * handler); -IVI_seg_t * IVI_last_seg(IVI_t * handler); -IVI_seg_t * IVI_prev_seg(IVI_seg_t * seg); -IVI_seg_t * IVI_next_seg(IVI_seg_t * seg); -IVI_seg_t * IVI_prev_continuous_seg(IVI_seg_t * seg); -IVI_seg_t * IVI_next_continuous_seg(IVI_seg_t * seg); - - -/** - * Relation of two segments - **/ -typedef enum __Relation_t{ - LEFT_NO_OVERLAP = 1, // |___A___| - // |___B___| - - LEFT_OVERLAP, // |___A___| - // |___B___| - - CONTAINED, // |___A___| - // |_____B_____| - - CONTAIN, // |_____A_____| - // |___B___| - - RIGHT_OVERLAP, // |___A___| - // |___B___| - - RIGHT_NO_OVERLAP, // |___A___| - // |___B___| - - ERROR -}Relation_t; - - -/** - * Name: - * IVI_relative_position - * Description: - * Get relative position of given two interval segments - * Params: - * seg1: Subject of relation - * seg2: Object of relation - * Relation: - * On success, return the relation of two segments with enum; - * Else, return ERROR in enum; - **/ -Relation_t IVI_relative_position(IVI_seg_t * seg1, IVI_seg_t * seg2); - - - -/** - * Name: - * IVI_create - * Description: - * Create an InterVal Index - * Params: - * void - * Return: - * Return a handler of this InterVal Index - **/ -IVI_t * IVI_create(void); - - - -/** - * Name: - * IVI_destroy - * Description: - * Destroy a given InterVal Index's handler - * Params: - * handler: The InterVal Index you want to destroy - * cb: Callback function for user to free data in segement - * usr_para: User parameter - * Return: - * void - **/ -void IVI_destroy(IVI_t * handler, IVI_callback_t cb, void * usr_para); - - - -/** - * Name: - * IVI_seg_malloc - * Description: - * Malloc a segment with given parameters - * Params: - * left: Left point of segment - * right: Right point of segment - * data: User data - * Return: - * Return a pointer of segment structure. - **/ -IVI_seg_t * IVI_seg_malloc(OFFSET_TYPE left, OFFSET_TYPE right, void * data); - - - -/** - * Name: - * IVI_seg_free - * Description: - * Free the memory of given segment - * Params: - * seg: The segment that you want to free - * cb: Callback function for user to free *data in seg - * usr_para: User parameter for cb - * Return: - * void - **/ -void IVI_seg_free(IVI_seg_t * seg, IVI_callback_t cb, void * usr_para); - - - -/** - * Name: - * IVI_insert - * Description: - * Insert a segment to an InterVal Index handler,and the segment - * MUST not be overlapped with others in handler. - * Params: - * handler: The handler of InterVal Index created by IVI_create - * seg: A segment that user wants to add. It MUST be created - * by IVI_seg_malloc. - * Return: - * On success, 0 is returned; - * Else when overlapp occures or error occures, -1 is returned. - **/ -int IVI_insert(IVI_t * handler, IVI_seg_t * seg); - - - -/** - * Name: - * IVI_remove - * Description: - * Remove a given segment from given InterVal Index handler. - * Params: - * handler: The handler of InterVal Index created by IVI_create - * seg: A segment that user wants to delete. It MUST be created - * by IVI_seg_malloc. - * Return: - * On success, 0 is returned; - * Else when overlapp occures, -1 is returned. - **/ -int IVI_remove(IVI_t * handler, IVI_seg_t * seg); - - - -/** - * Name: - * IVI_query - * Description: - * Query from given InterVal Index and get the number of segments - * which are overlapped with given interval, and store those segments - * in the last parameter. - * Params: - * handler: The handler of interval index created by IVI_create - * left: Left point of given interval - * right: Right point of given interval - * segs: An address of a segment pointer array to store those segments which - * are overlapped with given interval. NOTE that user should not malloc - * the array, and segs need to be freed by user. The element of *segs - * MUST not be freed by user. - * Return: - * Return the number of segments which are overlapped with given interval - **/ -int IVI_query(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs); - - - -/** - * Name: - * IVI_query_continuous - * Description: - * Query from interval index handler and get the number of continous segments - * which are overlapped with given interval. - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * left: Left point of given interval - * right: Right point of given interval - * segs: An address of a segment pointer array to store those segments which - * are overlapped with given interval. NOTE that user should not malloc - * the array, and segs need to be freed by user. The element of *segs - * MUST not be freed by user. - * Return: - * Return the number of continous segments which are overlapped with given interval - **/ -int IVI_query_continuous(IVI_t * handler, OFFSET_TYPE left, OFFSET_TYPE right, IVI_seg_t *** segs); - - - -/** - * Name: - * IVI_seg_cnt - * Description: - * Get the count of segments in given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the count of segments in given interval index handler - **/ -int IVI_seg_cnt(IVI_t * handler); - - -/** - * Name: - * IVI_seg_len - * Description: - * Get the length of whole segments in given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the length of whole segments in given interval index handler - **/ -OFFSET_TYPE IVI_seg_length(IVI_t * handler); - - -/** - * Name: - * IVI_mem_occupy - * Description: - * Get the memory occupy of given interval index handler - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * Return: - * Return the memory occupy of given interval index handler - **/ -unsigned long long IVI_mem_occupy(IVI_t * handler); - - -/** - * Name: - * IVI_traverse - * Description: - * Traverse given InterVal Index and execute given callback function - * one time for each seg in InterVal Index. - * Params: - * handler: The handler of InterVal Index created by IVI_create. - * IVI_callback_t: Callback function for user to define. - * usr_para: Parameter user want to pass to callback function. - * Return: - * void - **/ -void IVI_traverse(IVI_t * handler, IVI_callback_t cb, void * usr_para); - - - -#ifdef __cplusplus -} -#endif - -#endif /* _INTERVAL_INDEX_H_ */ diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h deleted file mode 100644 index 4c00193..0000000 --- a/src/inc_internal/json2iris.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef H_MAAT_JSON2IRIS_H_INCLUDE -#define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); -int set_file_rulenum(const char* path, int rulenum, void* logger); -#endif - diff --git a/src/inc_internal/map_str2int.h b/src/inc_internal/map_str2int.h deleted file mode 100644 index b9c53c6..0000000 --- a/src/inc_internal/map_str2int.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -struct maat_kv_store; -struct maat_kv_store* maat_kv_store_new(void); -void maat_kv_store_free(struct maat_kv_store* store); -int maat_kv_register(struct maat_kv_store* store, const char* key, int value); -int maat_kv_read(struct maat_kv_store* store, const char* key, int* value); -int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key_sz, int* value); -struct maat_kv_store* maat_kv_store_duplicate(struct maat_kv_store* store); - - diff --git a/src/inc_internal/queue.h b/src/inc_internal/queue.h deleted file mode 100644 index eec1e9b..0000000 --- a/src/inc_internal/queue.h +++ /dev/null @@ -1,113 +0,0 @@ -/*************************************************** -* TAILQ int Linux's -****************************************************/ - - -#ifndef _QUEUE_H_ -#define _QUEUE_H_ - -/** - * Tail queue definitions. - */ -#define _TAILQ_HEAD(name, type, qual) \ -struct name { \ - qual type *tqh_first; /* first element */ \ - qual type *qual *tqh_last; /* addr of last next element */ \ -} -#define TAILQ_HEAD(name, type) _TAILQ_HEAD(name, struct type,) - -#define TAILQ_HEAD_INITIALIZER(head) \ - { NULL, &(head).tqh_first } - -#define _TAILQ_ENTRY(type, qual) \ -struct { \ - qual type *tqe_next; /* next element */ \ - qual type *qual *tqe_prev; /* address of previous next element */\ -} -#define TAILQ_ENTRY(type) _TAILQ_ENTRY(struct type,) - -/* - * Tail queue functions. - */ -#define TAILQ_INIT(head) do { \ - (head)->tqh_first = NULL; \ - (head)->tqh_last = &(head)->tqh_first; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_INSERT_HEAD(head, elm, field) do { \ - if (((elm)->field.tqe_next = (head)->tqh_first) != NULL) \ - (head)->tqh_first->field.tqe_prev = \ - &(elm)->field.tqe_next; \ - else \ - (head)->tqh_last = &(elm)->field.tqe_next; \ - (head)->tqh_first = (elm); \ - (elm)->field.tqe_prev = &(head)->tqh_first; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_INSERT_TAIL(head, elm, field) do { \ - (elm)->field.tqe_next = NULL; \ - (elm)->field.tqe_prev = (head)->tqh_last; \ - *(head)->tqh_last = (elm); \ - (head)->tqh_last = &(elm)->field.tqe_next; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ - if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\ - (elm)->field.tqe_next->field.tqe_prev = \ - &(elm)->field.tqe_next; \ - else \ - (head)->tqh_last = &(elm)->field.tqe_next; \ - (listelm)->field.tqe_next = (elm); \ - (elm)->field.tqe_prev = &(listelm)->field.tqe_next; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ - (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ - (elm)->field.tqe_next = (listelm); \ - *(listelm)->field.tqe_prev = (elm); \ - (listelm)->field.tqe_prev = &(elm)->field.tqe_next; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_REMOVE(head, elm, field) do { \ - if (((elm)->field.tqe_next) != NULL) \ - (elm)->field.tqe_next->field.tqe_prev = \ - (elm)->field.tqe_prev; \ - else \ - (head)->tqh_last = (elm)->field.tqe_prev; \ - *(elm)->field.tqe_prev = (elm)->field.tqe_next; \ -} while (/*CONSTCOND*/0) - -#define TAILQ_FOREACH(var, head, field) \ - for ((var) = ((head)->tqh_first); \ - (var); \ - (var) = ((var)->field.tqe_next)) - -#define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ - for ((var) = (*(((struct headname *)((head)->tqh_last))->tqh_last)); \ - (var); \ - (var) = (*(((struct headname *)((var)->field.tqe_prev))->tqh_last))) - -#define TAILQ_CONCAT(head1, head2, field) do { \ - if (!TAILQ_EMPTY(head2)) { \ - *(head1)->tqh_last = (head2)->tqh_first; \ - (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ - (head1)->tqh_last = (head2)->tqh_last; \ - TAILQ_INIT((head2)); \ - } \ -} while (/*CONSTCOND*/0) - -/* - * Tail queue access methods. - */ -#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) -#define TAILQ_FIRST(head) ((head)->tqh_first) -#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) - -#define TAILQ_LAST(head, headname) \ - (*(((struct headname *)((head)->tqh_last))->tqh_last)) -#define TAILQ_PREV(elm, headname, field) \ - (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) - - - -#endif diff --git a/src/inc_internal/rbtree.h b/src/inc_internal/rbtree.h deleted file mode 100644 index adfad02..0000000 --- a/src/inc_internal/rbtree.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/include/linux/rbtree.h - - To use rbtrees you'll have to implement your own insert and search cores. - This will avoid us to use callbacks and to drop drammatically performances. - I know it's not the cleaner way, but in C (not in C++) to get - performances and genericity... - - See Documentation/rbtree.txt for documentation and samples. -*/ - -#ifndef _LINUX_RBTREE_H -#define _LINUX_RBTREE_H - -#include - -struct rb_node { - unsigned long __rb_parent_color; - struct rb_node *rb_right; - struct rb_node *rb_left; -} __attribute__((aligned(sizeof(long)))); - /* The alignment might seem pointless, but allegedly CRIS needs it */ - -struct rb_root { - struct rb_node *rb_node; -}; - - -#define rb_parent(r) ((struct rb_node *)((r)->__rb_parent_color & ~3)) - -#define RB_ROOT (struct rb_root) { NULL, } - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type, member) );}) -#endif - -#define rb_entry(ptr, type, member) container_of(ptr, type, member) - -#define RB_EMPTY_ROOT(root) ((root)->rb_node == NULL) - -/* 'empty' nodes are nodes that are known not to be inserted in an rbtree */ -#define RB_EMPTY_NODE(node) \ - ((node)->__rb_parent_color == (unsigned long)(node)) -#define RB_CLEAR_NODE(node) \ - ((node)->__rb_parent_color = (unsigned long)(node)) - - -extern void rb_insert_color(struct rb_node *, struct rb_root *); -extern void rb_erase(struct rb_node *, struct rb_root *); - - -/* Find logical next and previous nodes in a tree */ -extern struct rb_node *rb_next(const struct rb_node *); -extern struct rb_node *rb_prev(const struct rb_node *); -extern struct rb_node *rb_first(const struct rb_root *); -extern struct rb_node *rb_last(const struct rb_root *); - -/* Postorder iteration - always visit the parent after its children */ -extern struct rb_node *rb_first_postorder(const struct rb_root *); -extern struct rb_node *rb_next_postorder(const struct rb_node *); - -/* Fast replacement of a single node without remove/rebalance/add/rebalance */ -extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, - struct rb_root *root); - -static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, - struct rb_node ** rb_link) -{ - node->__rb_parent_color = (unsigned long)parent; - node->rb_left = node->rb_right = NULL; - - *rb_link = node; -} - -#define rb_entry_safe(ptr, type, member) \ - ({ typeof(ptr) ____ptr = (ptr); \ - ____ptr ? rb_entry(____ptr, type, member) : NULL; \ - }) - -/** - * rbtree_postorder_for_each_entry_safe - iterate over rb_root in post order of - * given type safe against removal of rb_node entry - * - * @pos: the 'type *' to use as a loop cursor. - * @n: another 'type *' to use as temporary storage - * @root: 'rb_root *' of the rbtree. - * @field: the name of the rb_node field within 'type'. - */ -#define rbtree_postorder_for_each_entry_safe(pos, n, root, field) \ - for (pos = rb_entry_safe(rb_first_postorder(root), typeof(*pos), field); \ - pos && ({ n = rb_entry_safe(rb_next_postorder(&pos->field), \ - typeof(*pos), field); 1; }); \ - pos = n) - -#endif /* _LINUX_RBTREE_H */ diff --git a/src/inc_internal/rbtree_augmented.h b/src/inc_internal/rbtree_augmented.h deleted file mode 100644 index bf3b639..0000000 --- a/src/inc_internal/rbtree_augmented.h +++ /dev/null @@ -1,241 +0,0 @@ -/* - Red Black Trees - (C) 1999 Andrea Arcangeli - (C) 2002 David Woodhouse - (C) 2012 Michel Lespinasse - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - linux/include/linux/rbtree_augmented.h -*/ - -#ifndef _LINUX_RBTREE_AUGMENTED_H -#define _LINUX_RBTREE_AUGMENTED_H - -#include "rbtree.h" - -/* - * Please note - only struct rb_augment_callbacks and the prototypes for - * rb_insert_augmented() and rb_erase_augmented() are intended to be public. - * The rest are implementation details you are not expected to depend on. - * - * See Documentation/rbtree.txt for documentation and samples. - */ - -struct rb_augment_callbacks { - void (*propagate)(struct rb_node *node, struct rb_node *stop); - void (*copy)(struct rb_node *old, struct rb_node *new); - void (*rotate)(struct rb_node *old, struct rb_node *new); -}; - -extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); -/* - * Fixup the rbtree and update the augmented information when rebalancing. - * - * On insertion, the user must update the augmented information on the path - * leading to the inserted node, then call rb_link_node() as usual and - * rb_augment_inserted() instead of the usual rb_insert_color() call. - * If rb_augment_inserted() rebalances the rbtree, it will callback into - * a user provided function to update the augmented information on the - * affected subtrees. - */ -static inline void -rb_insert_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - __rb_insert_augmented(node, root, augment->rotate); -} - -#define RB_DECLARE_CALLBACKS(rbstatic, rbname, rbstruct, rbfield, \ - rbtype, rbaugmented, rbcompute) \ -static inline void \ -rbname ## _propagate(struct rb_node *rb, struct rb_node *stop) \ -{ \ - while (rb != stop) { \ - rbstruct *node = rb_entry(rb, rbstruct, rbfield); \ - rbtype augmented = rbcompute(node); \ - if (node->rbaugmented == augmented) \ - break; \ - node->rbaugmented = augmented; \ - rb = rb_parent(&node->rbfield); \ - } \ -} \ -static inline void \ -rbname ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ -{ \ - rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ - rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ - new->rbaugmented = old->rbaugmented; \ -} \ -static void \ -rbname ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ -{ \ - rbstruct *old = rb_entry(rb_old, rbstruct, rbfield); \ - rbstruct *new = rb_entry(rb_new, rbstruct, rbfield); \ - new->rbaugmented = old->rbaugmented; \ - old->rbaugmented = rbcompute(old); \ -} \ -rbstatic const struct rb_augment_callbacks rbname = { \ - rbname ## _propagate, rbname ## _copy, rbname ## _rotate \ -}; - - -#define RB_RED 0 -#define RB_BLACK 1 - -#define __rb_parent(pc) ((struct rb_node *)(pc & ~3)) - -#define __rb_color(pc) ((pc) & 1) -#define __rb_is_black(pc) __rb_color(pc) -#define __rb_is_red(pc) (!__rb_color(pc)) -#define rb_color(rb) __rb_color((rb)->__rb_parent_color) -#define rb_is_red(rb) __rb_is_red((rb)->__rb_parent_color) -#define rb_is_black(rb) __rb_is_black((rb)->__rb_parent_color) - -static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p) -{ - rb->__rb_parent_color = rb_color(rb) | (unsigned long)p; -} - -static inline void rb_set_parent_color(struct rb_node *rb, - struct rb_node *p, int color) -{ - rb->__rb_parent_color = (unsigned long)p | color; -} - -static inline void -__rb_change_child(struct rb_node *old, struct rb_node *new, - struct rb_node *parent, struct rb_root *root) -{ - if (parent) { - if (parent->rb_left == old) - parent->rb_left = new; - else - parent->rb_right = new; - } else - root->rb_node = new; -} - -extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, - void (*augment_rotate)(struct rb_node *old, struct rb_node *new)); - -static __always_inline struct rb_node * -__rb_erase_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - struct rb_node *child = node->rb_right, *tmp = node->rb_left; - struct rb_node *parent, *rebalance; - unsigned long pc; - - if (!tmp) { - /* - * Case 1: node to erase has no more than 1 child (easy!) - * - * Note that if there is one child it must be red due to 5) - * and node must be black due to 4). We adjust colors locally - * so as to bypass __rb_erase_color() later on. - */ - pc = node->__rb_parent_color; - parent = __rb_parent(pc); - __rb_change_child(node, child, parent, root); - if (child) { - child->__rb_parent_color = pc; - rebalance = NULL; - } else - rebalance = __rb_is_black(pc) ? parent : NULL; - tmp = parent; - } else if (!child) { - /* Still case 1, but this time the child is node->rb_left */ - tmp->__rb_parent_color = pc = node->__rb_parent_color; - parent = __rb_parent(pc); - __rb_change_child(node, tmp, parent, root); - rebalance = NULL; - tmp = parent; - } else { - struct rb_node *successor = child, *child2; - tmp = child->rb_left; - if (!tmp) { - /* - * Case 2: node's successor is its right child - * - * (n) (s) - * / \ / \ - * (x) (s) -> (x) (c) - * \ - * (c) - */ - parent = successor; - child2 = successor->rb_right; - augment->copy(node, successor); - } else { - /* - * Case 3: node's successor is leftmost under - * node's right child subtree - * - * (n) (s) - * / \ / \ - * (x) (y) -> (x) (y) - * / / - * (p) (p) - * / / - * (s) (c) - * \ - * (c) - */ - do { - parent = successor; - successor = tmp; - tmp = tmp->rb_left; - } while (tmp); - parent->rb_left = child2 = successor->rb_right; - successor->rb_right = child; - rb_set_parent(child, successor); - augment->copy(node, successor); - augment->propagate(parent, successor); - } - - successor->rb_left = tmp = node->rb_left; - rb_set_parent(tmp, successor); - - pc = node->__rb_parent_color; - tmp = __rb_parent(pc); - __rb_change_child(node, successor, tmp, root); - if (child2) { - successor->__rb_parent_color = pc; - rb_set_parent_color(child2, parent, RB_BLACK); - rebalance = NULL; - } else { - unsigned long pc2 = successor->__rb_parent_color; - successor->__rb_parent_color = pc; - rebalance = __rb_is_black(pc2) ? parent : NULL; - } - tmp = successor; - } - - augment->propagate(tmp, NULL); - return rebalance; -} - -static __always_inline void -rb_erase_augmented(struct rb_node *node, struct rb_root *root, - const struct rb_augment_callbacks *augment) -{ - struct rb_node *rebalance = __rb_erase_augmented(node, root, augment); - if (rebalance) - __rb_erase_color(rebalance, root, augment->rotate); -} - -#endif /* _LINUX_RBTREE_AUGMENTED_H */ diff --git a/src/inc_internal/sfh_internal.h b/src/inc_internal/sfh_internal.h deleted file mode 100644 index 83c8f13..0000000 --- a/src/inc_internal/sfh_internal.h +++ /dev/null @@ -1,108 +0,0 @@ -#include "zt_hash.h" -#include "interval_index.h" -#include "stream_fuzzy_hash.h" - -#ifndef __SFH_INTERNAL_H_INCLUDE_ -#define __SFH_INTERNAL_H_INCLUDE_ - -#define ROLLING_WINDOW 7 -#define BLOCKSIZE_MIN 3 -#define HASH_PRIME 0x01000193 -#define HASH_INIT 0x28021967 -#define CALCULATE 0 -#define MODIFY 1 -#define EXPECT_SIGNATURE_LEN 64 -#define MEMORY_OCCUPY 3 - -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif - -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) -#endif - -#define DEBUG (0) - -//int count = 0; -struct roll_state_t -{ - unsigned char window[ROLLING_WINDOW]; - unsigned char pad[1]; - unsigned int h1, h2, h3; - unsigned int n; -}; - -typedef struct -{ - char mbuf[ROLLING_WINDOW-1]; - char pad[8-ROLLING_WINDOW+1]; - int slice_num; - unsigned int msize; - - struct zt_state_t p_state; //partial strong hash value - struct zt_state_t s_state; //strong hash state - struct roll_state_t r_state; - - unsigned long long left_offset; - unsigned long long right_offset; - - unsigned int * r_array; //array to store rolling hash value - unsigned int r_cnt; - unsigned int r_size; - struct zt_state_t * s_array; //array to store strong(Tillichi-Zemor) hash value - unsigned int s_cnt; //always point to the next available position - unsigned int s_size; -}sfh_seg_t; - - -typedef struct -{ - unsigned long long orilen; - IVI_t * ivi; //ÿһ¸öhandleÀïÃæ±£´æÒ»¸öIVIÖ¸Õ룬һ¸öIVIÀïÃæ±£´æµÄÊÇÒ»¸öÎļþÀïµÄƬ - unsigned long long effective_length; - unsigned long long blocksize; - unsigned long long fuzzy_node_memory; - unsigned long long IVI_memory; - unsigned long long length_increase; - int s_state_cnt; - unsigned int sim_tuned_rs_cnt;//rolling state count after a tune simulation - int do_tune; -}fuzzy_handle_inner_t; - - -typedef struct -{ - char * hash_b1; //×îºóÊä³ö½á¹ûµÄcharÊý×é - char * hash_b2; - unsigned int size_b1,size_b2; - unsigned int offset_b1,offset_b2; //Êý×鳤¶È - unsigned long long first_ZTH_offset; - unsigned long long last_ZTH_offset; - char last_char_b1, last_char_b2; - unsigned long long b1,b2; -}sfh_output_t; - - -typedef struct -{ - unsigned long long first_ZTH_offset; - unsigned long long last_ZTH_offset; - unsigned long long hash_length; -}final_length; - -int destroy_sfh_seg(sfh_seg_t*p); -unsigned long long get_blocksize(unsigned long long orilen); -int sfh_merge_seg(fuzzy_handle_inner_t * _handle,sfh_seg_t * seg, sfh_seg_t * next_seg, unsigned long long blocksize); -int sfh_update_seg(fuzzy_handle_inner_t * _handle,sfh_seg_t * p, const char * data, unsigned long data_size, unsigned long long blocksize); -unsigned int segment_overlap(fuzzy_handle_inner_t * handle, unsigned int size, unsigned long long offset, const char * data); -void sfh_tune_callback(IVI_seg_t * seg, void * user_para); -void sfh_output_callback(IVI_seg_t * seg, void * user_para); -void fuzzy_hash_length(IVI_seg_t * seg, void * user_para); -#endif - diff --git a/src/inc_internal/uthash/utarray.h b/src/inc_internal/uthash/utarray.h deleted file mode 100644 index 6b62018..0000000 --- a/src/inc_internal/uthash/utarray.h +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright (c) 2008-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* a dynamic array implementation using macros - */ -#ifndef UTARRAY_H -#define UTARRAY_H - -#define UTARRAY_VERSION 2.1.0 - -#include /* size_t */ -#include /* memset, etc */ -#include /* exit */ - -#ifdef __GNUC__ -#define UTARRAY_UNUSED __attribute__((__unused__)) -#else -#define UTARRAY_UNUSED -#endif - -#ifdef oom -#error "The name of macro 'oom' has been changed to 'utarray_oom'. Please update your code." -#define utarray_oom() oom() -#endif - -#ifndef utarray_oom -#define utarray_oom() exit(-1) -#endif - -typedef void (ctor_f)(void *dst, const void *src); -typedef void (dtor_f)(void *elt); -typedef void (init_f)(void *elt); -typedef struct { - size_t sz; - init_f *init; - ctor_f *copy; - dtor_f *dtor; -} UT_icd; - -typedef struct { - unsigned i,n;/* i: index of next available slot, n: num slots */ - UT_icd icd; /* initializer, copy and destructor functions */ - char *d; /* n slots of size icd->sz*/ -} UT_array; - -#define utarray_init(a,_icd) do { \ - memset(a,0,sizeof(UT_array)); \ - (a)->icd = *(_icd); \ -} while(0) - -#define utarray_done(a) do { \ - if ((a)->n) { \ - if ((a)->icd.dtor) { \ - unsigned _ut_i; \ - for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ - (a)->icd.dtor(utarray_eltptr(a,_ut_i)); \ - } \ - } \ - free((a)->d); \ - } \ - (a)->n=0; \ -} while(0) - -#define utarray_new(a,_icd) do { \ - (a) = (UT_array*)malloc(sizeof(UT_array)); \ - if ((a) == NULL) { \ - utarray_oom(); \ - } \ - utarray_init(a,_icd); \ -} while(0) - -#define utarray_free(a) do { \ - utarray_done(a); \ - free(a); \ -} while(0) - -#define utarray_reserve(a,by) do { \ - if (((a)->i+(by)) > (a)->n) { \ - char *utarray_tmp; \ - while (((a)->i+(by)) > (a)->n) { (a)->n = ((a)->n ? (2*(a)->n) : 8); } \ - utarray_tmp=(char*)realloc((a)->d, (a)->n*(a)->icd.sz); \ - if (utarray_tmp == NULL) { \ - utarray_oom(); \ - } \ - (a)->d=utarray_tmp; \ - } \ -} while(0) - -#define utarray_push_back(a,p) do { \ - utarray_reserve(a,1); \ - if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,(a)->i++), p); } \ - else { memcpy(_utarray_eltptr(a,(a)->i++), p, (a)->icd.sz); }; \ -} while(0) - -#define utarray_pop_back(a) do { \ - if ((a)->icd.dtor) { (a)->icd.dtor( _utarray_eltptr(a,--((a)->i))); } \ - else { (a)->i--; } \ -} while(0) - -#define utarray_extend_back(a) do { \ - utarray_reserve(a,1); \ - if ((a)->icd.init) { (a)->icd.init(_utarray_eltptr(a,(a)->i)); } \ - else { memset(_utarray_eltptr(a,(a)->i),0,(a)->icd.sz); } \ - (a)->i++; \ -} while(0) - -#define utarray_len(a) ((a)->i) - -#define utarray_eltptr(a,j) (((j) < (a)->i) ? _utarray_eltptr(a,j) : NULL) -#define _utarray_eltptr(a,j) ((void*)((a)->d + ((a)->icd.sz * (j)))) - -#define utarray_insert(a,p,j) do { \ - if ((j) > (a)->i) utarray_resize(a,j); \ - utarray_reserve(a,1); \ - if ((j) < (a)->i) { \ - memmove( _utarray_eltptr(a,(j)+1), _utarray_eltptr(a,j), \ - ((a)->i - (j))*((a)->icd.sz)); \ - } \ - if ((a)->icd.copy) { (a)->icd.copy( _utarray_eltptr(a,j), p); } \ - else { memcpy(_utarray_eltptr(a,j), p, (a)->icd.sz); }; \ - (a)->i++; \ -} while(0) - -#define utarray_inserta(a,w,j) do { \ - if (utarray_len(w) == 0) break; \ - if ((j) > (a)->i) utarray_resize(a,j); \ - utarray_reserve(a,utarray_len(w)); \ - if ((j) < (a)->i) { \ - memmove(_utarray_eltptr(a,(j)+utarray_len(w)), \ - _utarray_eltptr(a,j), \ - ((a)->i - (j))*((a)->icd.sz)); \ - } \ - if ((a)->icd.copy) { \ - unsigned _ut_i; \ - for(_ut_i=0;_ut_i<(w)->i;_ut_i++) { \ - (a)->icd.copy(_utarray_eltptr(a, (j) + _ut_i), _utarray_eltptr(w, _ut_i)); \ - } \ - } else { \ - memcpy(_utarray_eltptr(a,j), _utarray_eltptr(w,0), \ - utarray_len(w)*((a)->icd.sz)); \ - } \ - (a)->i += utarray_len(w); \ -} while(0) - -#define utarray_resize(dst,num) do { \ - unsigned _ut_i; \ - if ((dst)->i > (unsigned)(num)) { \ - if ((dst)->icd.dtor) { \ - for (_ut_i = (num); _ut_i < (dst)->i; ++_ut_i) { \ - (dst)->icd.dtor(_utarray_eltptr(dst, _ut_i)); \ - } \ - } \ - } else if ((dst)->i < (unsigned)(num)) { \ - utarray_reserve(dst, (num) - (dst)->i); \ - if ((dst)->icd.init) { \ - for (_ut_i = (dst)->i; _ut_i < (unsigned)(num); ++_ut_i) { \ - (dst)->icd.init(_utarray_eltptr(dst, _ut_i)); \ - } \ - } else { \ - memset(_utarray_eltptr(dst, (dst)->i), 0, (dst)->icd.sz*((num) - (dst)->i)); \ - } \ - } \ - (dst)->i = (num); \ -} while(0) - -#define utarray_concat(dst,src) do { \ - utarray_inserta(dst, src, utarray_len(dst)); \ -} while(0) - -#define utarray_erase(a,pos,len) do { \ - if ((a)->icd.dtor) { \ - unsigned _ut_i; \ - for (_ut_i = 0; _ut_i < (len); _ut_i++) { \ - (a)->icd.dtor(utarray_eltptr(a, (pos) + _ut_i)); \ - } \ - } \ - if ((a)->i > ((pos) + (len))) { \ - memmove(_utarray_eltptr(a, pos), _utarray_eltptr(a, (pos) + (len)), \ - ((a)->i - ((pos) + (len))) * (a)->icd.sz); \ - } \ - (a)->i -= (len); \ -} while(0) - -#define utarray_renew(a,u) do { \ - if (a) utarray_clear(a); \ - else utarray_new(a, u); \ -} while(0) - -#define utarray_clear(a) do { \ - if ((a)->i > 0) { \ - if ((a)->icd.dtor) { \ - unsigned _ut_i; \ - for(_ut_i=0; _ut_i < (a)->i; _ut_i++) { \ - (a)->icd.dtor(_utarray_eltptr(a, _ut_i)); \ - } \ - } \ - (a)->i = 0; \ - } \ -} while(0) - -#define utarray_sort(a,cmp) do { \ - qsort((a)->d, (a)->i, (a)->icd.sz, cmp); \ -} while(0) - -#define utarray_find(a,v,cmp) bsearch((v),(a)->d,(a)->i,(a)->icd.sz,cmp) - -#define utarray_front(a) (((a)->i) ? (_utarray_eltptr(a,0)) : NULL) -#define utarray_next(a,e) (((e)==NULL) ? utarray_front(a) : (((a)->i != utarray_eltidx(a,e)+1) ? _utarray_eltptr(a,utarray_eltidx(a,e)+1) : NULL)) -#define utarray_prev(a,e) (((e)==NULL) ? utarray_back(a) : ((utarray_eltidx(a,e) != 0) ? _utarray_eltptr(a,utarray_eltidx(a,e)-1) : NULL)) -#define utarray_back(a) (((a)->i) ? (_utarray_eltptr(a,(a)->i-1)) : NULL) -#define utarray_eltidx(a,e) (((char*)(e) - (a)->d) / (a)->icd.sz) - -/* last we pre-define a few icd for common utarrays of ints and strings */ -static void utarray_str_cpy(void *dst, const void *src) { - char **_src = (char**)src, **_dst = (char**)dst; - *_dst = (*_src == NULL) ? NULL : strdup(*_src); -} -static void utarray_str_dtor(void *elt) { - char **eltc = (char**)elt; - if (*eltc != NULL) free(*eltc); -} -static const UT_icd ut_str_icd UTARRAY_UNUSED = {sizeof(char*),NULL,utarray_str_cpy,utarray_str_dtor}; -static const UT_icd ut_int_icd UTARRAY_UNUSED = {sizeof(int),NULL,NULL,NULL}; -static const UT_icd ut_ptr_icd UTARRAY_UNUSED = {sizeof(void*),NULL,NULL,NULL}; - - -#endif /* UTARRAY_H */ diff --git a/src/inc_internal/uthash/uthash.h b/src/inc_internal/uthash/uthash.h deleted file mode 100644 index 5e5866a..0000000 --- a/src/inc_internal/uthash/uthash.h +++ /dev/null @@ -1,1150 +0,0 @@ -/* -Copyright (c) 2003-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef UTHASH_H -#define UTHASH_H - -#define UTHASH_VERSION 2.1.0 - -#include /* memcmp, memset, strlen */ -#include /* ptrdiff_t */ -#include /* exit */ - -/* These macros use decltype or the earlier __typeof GNU extension. - As decltype is only available in newer compilers (VS2010 or gcc 4.3+ - when compiling c++ source) this code uses whatever method is needed - or, for VS2008 where neither is available, uses casting workarounds. */ -#if !defined(DECLTYPE) && !defined(NO_DECLTYPE) -#if defined(_MSC_VER) /* MS compiler */ -#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ -#define DECLTYPE(x) (decltype(x)) -#else /* VS2008 or older (or VS2010 in C mode) */ -#define NO_DECLTYPE -#endif -#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__) -#define NO_DECLTYPE -#else /* GNU, Sun and other compilers */ -#define DECLTYPE(x) (__typeof(x)) -#endif -#endif - -#ifdef NO_DECLTYPE -#define DECLTYPE(x) -#define DECLTYPE_ASSIGN(dst,src) \ -do { \ - char **_da_dst = (char**)(&(dst)); \ - *_da_dst = (char*)(src); \ -} while (0) -#else -#define DECLTYPE_ASSIGN(dst,src) \ -do { \ - (dst) = DECLTYPE(dst)(src); \ -} while (0) -#endif - -/* a number of the hash function use uint32_t which isn't defined on Pre VS2010 */ -#if defined(_WIN32) -#if defined(_MSC_VER) && _MSC_VER >= 1600 -#include -#elif defined(__WATCOMC__) || defined(__MINGW32__) || defined(__CYGWIN__) -#include -#else -typedef unsigned int uint32_t; -typedef unsigned char uint8_t; -#endif -#elif defined(__GNUC__) && !defined(__VXWORKS__) -#include -#else -typedef unsigned int uint32_t; -typedef unsigned char uint8_t; -#endif - -#ifndef uthash_malloc -#define uthash_malloc(sz) malloc(sz) /* malloc fcn */ -#endif -#ifndef uthash_free -#define uthash_free(ptr,sz) free(ptr) /* free fcn */ -#endif -#ifndef uthash_bzero -#define uthash_bzero(a,n) memset(a,'\0',n) -#endif -#ifndef uthash_strlen -#define uthash_strlen(s) strlen(s) -#endif - -#ifdef uthash_memcmp -/* This warning will not catch programs that define uthash_memcmp AFTER including uthash.h. */ -#warning "uthash_memcmp is deprecated; please use HASH_KEYCMP instead" -#else -#define uthash_memcmp(a,b,n) memcmp(a,b,n) -#endif - -#ifndef HASH_KEYCMP -#define HASH_KEYCMP(a,b,n) uthash_memcmp(a,b,n) -#endif - -#ifndef uthash_noexpand_fyi -#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */ -#endif -#ifndef uthash_expand_fyi -#define uthash_expand_fyi(tbl) /* can be defined to log expands */ -#endif - -#ifndef HASH_NONFATAL_OOM -#define HASH_NONFATAL_OOM 0 -#endif - -#if HASH_NONFATAL_OOM -/* malloc failures can be recovered from */ - -#ifndef uthash_nonfatal_oom -#define uthash_nonfatal_oom(obj) do {} while (0) /* non-fatal OOM error */ -#endif - -#define HASH_RECORD_OOM(oomed) do { (oomed) = 1; } while (0) -#define IF_HASH_NONFATAL_OOM(x) x - -#else -/* malloc failures result in lost memory, hash tables are unusable */ - -#ifndef uthash_fatal -#define uthash_fatal(msg) exit(-1) /* fatal OOM error */ -#endif - -#define HASH_RECORD_OOM(oomed) uthash_fatal("out of memory") -#define IF_HASH_NONFATAL_OOM(x) - -#endif - -/* initial number of buckets */ -#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ -#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */ -#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ - -/* calculate the element whose hash handle address is hhp */ -#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho))) -/* calculate the hash handle from element address elp */ -#define HH_FROM_ELMT(tbl,elp) ((UT_hash_handle*)(void*)(((char*)(elp)) + ((tbl)->hho))) - -#define HASH_ROLLBACK_BKT(hh, head, itemptrhh) \ -do { \ - struct UT_hash_handle *_hd_hh_item = (itemptrhh); \ - unsigned _hd_bkt; \ - HASH_TO_BKT(_hd_hh_item->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ - (head)->hh.tbl->buckets[_hd_bkt].count++; \ - _hd_hh_item->hh_next = NULL; \ - _hd_hh_item->hh_prev = NULL; \ -} while (0) - -#define HASH_VALUE(keyptr,keylen,hashv) \ -do { \ - HASH_FCN(keyptr, keylen, hashv); \ -} while (0) - -#define HASH_FIND_BYHASHVALUE(hh,head,keyptr,keylen,hashval,out) \ -do { \ - (out) = NULL; \ - if (head) { \ - unsigned _hf_bkt; \ - HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \ - if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) { \ - HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \ - } \ - } \ -} while (0) - -#define HASH_FIND(hh,head,keyptr,keylen,out) \ -do { \ - (out) = NULL; \ - if (head) { \ - unsigned _hf_hashv; \ - HASH_VALUE(keyptr, keylen, _hf_hashv); \ - HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out); \ - } \ -} while (0) - -#ifdef HASH_BLOOM -#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM) -#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8UL) + (((HASH_BLOOM_BITLEN%8UL)!=0UL) ? 1UL : 0UL) -#define HASH_BLOOM_MAKE(tbl,oomed) \ -do { \ - (tbl)->bloom_nbits = HASH_BLOOM; \ - (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \ - if (!(tbl)->bloom_bv) { \ - HASH_RECORD_OOM(oomed); \ - } else { \ - uthash_bzero((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ - (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \ - } \ -} while (0) - -#define HASH_BLOOM_FREE(tbl) \ -do { \ - uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ -} while (0) - -#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U))) -#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U))) - -#define HASH_BLOOM_ADD(tbl,hashv) \ - HASH_BLOOM_BITSET((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) - -#define HASH_BLOOM_TEST(tbl,hashv) \ - HASH_BLOOM_BITTEST((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) - -#else -#define HASH_BLOOM_MAKE(tbl,oomed) -#define HASH_BLOOM_FREE(tbl) -#define HASH_BLOOM_ADD(tbl,hashv) -#define HASH_BLOOM_TEST(tbl,hashv) (1) -#define HASH_BLOOM_BYTELEN 0U -#endif - -#define HASH_MAKE_TABLE(hh,head,oomed) \ -do { \ - (head)->hh.tbl = (UT_hash_table*)uthash_malloc(sizeof(UT_hash_table)); \ - if (!(head)->hh.tbl) { \ - HASH_RECORD_OOM(oomed); \ - } else { \ - uthash_bzero((head)->hh.tbl, sizeof(UT_hash_table)); \ - (head)->hh.tbl->tail = &((head)->hh); \ - (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \ - (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \ - (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \ - (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \ - HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket)); \ - (head)->hh.tbl->signature = HASH_SIGNATURE; \ - if (!(head)->hh.tbl->buckets) { \ - HASH_RECORD_OOM(oomed); \ - uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ - } else { \ - uthash_bzero((head)->hh.tbl->buckets, \ - HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket)); \ - HASH_BLOOM_MAKE((head)->hh.tbl, oomed); \ - IF_HASH_NONFATAL_OOM( \ - if (oomed) { \ - uthash_free((head)->hh.tbl->buckets, \ - HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ - uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ - } \ - ) \ - } \ - } \ -} while (0) - -#define HASH_REPLACE_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,replaced,cmpfcn) \ -do { \ - (replaced) = NULL; \ - HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ - if (replaced) { \ - HASH_DELETE(hh, head, replaced); \ - } \ - HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn); \ -} while (0) - -#define HASH_REPLACE_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add,replaced) \ -do { \ - (replaced) = NULL; \ - HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ - if (replaced) { \ - HASH_DELETE(hh, head, replaced); \ - } \ - HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add); \ -} while (0) - -#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \ -do { \ - unsigned _hr_hashv; \ - HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ - HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced); \ -} while (0) - -#define HASH_REPLACE_INORDER(hh,head,fieldname,keylen_in,add,replaced,cmpfcn) \ -do { \ - unsigned _hr_hashv; \ - HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ - HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced, cmpfcn); \ -} while (0) - -#define HASH_APPEND_LIST(hh, head, add) \ -do { \ - (add)->hh.next = NULL; \ - (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \ - (head)->hh.tbl->tail->next = (add); \ - (head)->hh.tbl->tail = &((add)->hh); \ -} while (0) - -#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn) \ -do { \ - do { \ - if (cmpfcn(DECLTYPE(head)(_hs_iter), add) > 0) { \ - break; \ - } \ - } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ -} while (0) - -#ifdef NO_DECLTYPE -#undef HASH_AKBI_INNER_LOOP -#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn) \ -do { \ - char *_hs_saved_head = (char*)(head); \ - do { \ - DECLTYPE_ASSIGN(head, _hs_iter); \ - if (cmpfcn(head, add) > 0) { \ - DECLTYPE_ASSIGN(head, _hs_saved_head); \ - break; \ - } \ - DECLTYPE_ASSIGN(head, _hs_saved_head); \ - } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ -} while (0) -#endif - -#if HASH_NONFATAL_OOM - -#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed) \ -do { \ - if (!(oomed)) { \ - unsigned _ha_bkt; \ - (head)->hh.tbl->num_items++; \ - HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ - HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed); \ - if (oomed) { \ - HASH_ROLLBACK_BKT(hh, head, &(add)->hh); \ - HASH_DELETE_HH(hh, head, &(add)->hh); \ - (add)->hh.tbl = NULL; \ - uthash_nonfatal_oom(add); \ - } else { \ - HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ - HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ - } \ - } else { \ - (add)->hh.tbl = NULL; \ - uthash_nonfatal_oom(add); \ - } \ -} while (0) - -#else - -#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed) \ -do { \ - unsigned _ha_bkt; \ - (head)->hh.tbl->num_items++; \ - HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ - HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed); \ - HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ - HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ -} while (0) - -#endif - - -#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh,head,keyptr,keylen_in,hashval,add,cmpfcn) \ -do { \ - IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; ) \ - (add)->hh.hashv = (hashval); \ - (add)->hh.key = (char*) (keyptr); \ - (add)->hh.keylen = (unsigned) (keylen_in); \ - if (!(head)) { \ - (add)->hh.next = NULL; \ - (add)->hh.prev = NULL; \ - HASH_MAKE_TABLE(hh, add, _ha_oomed); \ - IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { ) \ - (head) = (add); \ - IF_HASH_NONFATAL_OOM( } ) \ - } else { \ - void *_hs_iter = (head); \ - (add)->hh.tbl = (head)->hh.tbl; \ - HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn); \ - if (_hs_iter) { \ - (add)->hh.next = _hs_iter; \ - if (((add)->hh.prev = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev)) { \ - HH_FROM_ELMT((head)->hh.tbl, (add)->hh.prev)->next = (add); \ - } else { \ - (head) = (add); \ - } \ - HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev = (add); \ - } else { \ - HASH_APPEND_LIST(hh, head, add); \ - } \ - } \ - HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed); \ - HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE_INORDER"); \ -} while (0) - -#define HASH_ADD_KEYPTR_INORDER(hh,head,keyptr,keylen_in,add,cmpfcn) \ -do { \ - unsigned _hs_hashv; \ - HASH_VALUE(keyptr, keylen_in, _hs_hashv); \ - HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, _hs_hashv, add, cmpfcn); \ -} while (0) - -#define HASH_ADD_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,cmpfcn) \ - HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn) - -#define HASH_ADD_INORDER(hh,head,fieldname,keylen_in,add,cmpfcn) \ - HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, cmpfcn) - -#define HASH_ADD_KEYPTR_BYHASHVALUE(hh,head,keyptr,keylen_in,hashval,add) \ -do { \ - IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; ) \ - (add)->hh.hashv = (hashval); \ - (add)->hh.key = (char*) (keyptr); \ - (add)->hh.keylen = (unsigned) (keylen_in); \ - if (!(head)) { \ - (add)->hh.next = NULL; \ - (add)->hh.prev = NULL; \ - HASH_MAKE_TABLE(hh, add, _ha_oomed); \ - IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { ) \ - (head) = (add); \ - IF_HASH_NONFATAL_OOM( } ) \ - } else { \ - (add)->hh.tbl = (head)->hh.tbl; \ - HASH_APPEND_LIST(hh, head, add); \ - } \ - HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed); \ - HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE"); \ -} while (0) - -#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \ -do { \ - unsigned _ha_hashv; \ - HASH_VALUE(keyptr, keylen_in, _ha_hashv); \ - HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, add); \ -} while (0) - -#define HASH_ADD_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add) \ - HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add) - -#define HASH_ADD(hh,head,fieldname,keylen_in,add) \ - HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add) - -#define HASH_TO_BKT(hashv,num_bkts,bkt) \ -do { \ - bkt = ((hashv) & ((num_bkts) - 1U)); \ -} while (0) - -/* delete "delptr" from the hash table. - * "the usual" patch-up process for the app-order doubly-linked-list. - * The use of _hd_hh_del below deserves special explanation. - * These used to be expressed using (delptr) but that led to a bug - * if someone used the same symbol for the head and deletee, like - * HASH_DELETE(hh,users,users); - * We want that to work, but by changing the head (users) below - * we were forfeiting our ability to further refer to the deletee (users) - * in the patch-up process. Solution: use scratch space to - * copy the deletee pointer, then the latter references are via that - * scratch pointer rather than through the repointed (users) symbol. - */ -#define HASH_DELETE(hh,head,delptr) \ - HASH_DELETE_HH(hh, head, &(delptr)->hh) - -#define HASH_DELETE_HH(hh,head,delptrhh) \ -do { \ - struct UT_hash_handle *_hd_hh_del = (delptrhh); \ - if ((_hd_hh_del->prev == NULL) && (_hd_hh_del->next == NULL)) { \ - HASH_BLOOM_FREE((head)->hh.tbl); \ - uthash_free((head)->hh.tbl->buckets, \ - (head)->hh.tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ - uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ - (head) = NULL; \ - } else { \ - unsigned _hd_bkt; \ - if (_hd_hh_del == (head)->hh.tbl->tail) { \ - (head)->hh.tbl->tail = HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev); \ - } \ - if (_hd_hh_del->prev != NULL) { \ - HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev)->next = _hd_hh_del->next; \ - } else { \ - DECLTYPE_ASSIGN(head, _hd_hh_del->next); \ - } \ - if (_hd_hh_del->next != NULL) { \ - HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->next)->prev = _hd_hh_del->prev; \ - } \ - HASH_TO_BKT(_hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ - HASH_DEL_IN_BKT((head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \ - (head)->hh.tbl->num_items--; \ - } \ - HASH_FSCK(hh, head, "HASH_DELETE_HH"); \ -} while (0) - -/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */ -#define HASH_FIND_STR(head,findstr,out) \ -do { \ - unsigned _uthash_hfstr_keylen = (unsigned)uthash_strlen(findstr); \ - HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out); \ -} while (0) -#define HASH_ADD_STR(head,strfield,add) \ -do { \ - unsigned _uthash_hastr_keylen = (unsigned)uthash_strlen((add)->strfield); \ - HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add); \ -} while (0) -#define HASH_REPLACE_STR(head,strfield,add,replaced) \ -do { \ - unsigned _uthash_hrstr_keylen = (unsigned)uthash_strlen((add)->strfield); \ - HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, replaced); \ -} while (0) -#define HASH_FIND_INT(head,findint,out) \ - HASH_FIND(hh,head,findint,sizeof(int),out) -#define HASH_ADD_INT(head,intfield,add) \ - HASH_ADD(hh,head,intfield,sizeof(int),add) -#define HASH_REPLACE_INT(head,intfield,add,replaced) \ - HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced) -#define HASH_FIND_PTR(head,findptr,out) \ - HASH_FIND(hh,head,findptr,sizeof(void *),out) -#define HASH_ADD_PTR(head,ptrfield,add) \ - HASH_ADD(hh,head,ptrfield,sizeof(void *),add) -#define HASH_REPLACE_PTR(head,ptrfield,add,replaced) \ - HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced) -#define HASH_DEL(head,delptr) \ - HASH_DELETE(hh,head,delptr) - -/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined. - * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined. - */ -#ifdef HASH_DEBUG -#include /* fprintf, stderr */ -#define HASH_OOPS(...) do { fprintf(stderr, __VA_ARGS__); exit(-1); } while (0) -#define HASH_FSCK(hh,head,where) \ -do { \ - struct UT_hash_handle *_thh; \ - if (head) { \ - unsigned _bkt_i; \ - unsigned _count = 0; \ - char *_prev; \ - for (_bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; ++_bkt_i) { \ - unsigned _bkt_count = 0; \ - _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \ - _prev = NULL; \ - while (_thh) { \ - if (_prev != (char*)(_thh->hh_prev)) { \ - HASH_OOPS("%s: invalid hh_prev %p, actual %p\n", \ - (where), (void*)_thh->hh_prev, (void*)_prev); \ - } \ - _bkt_count++; \ - _prev = (char*)(_thh); \ - _thh = _thh->hh_next; \ - } \ - _count += _bkt_count; \ - if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \ - HASH_OOPS("%s: invalid bucket count %u, actual %u\n", \ - (where), (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \ - } \ - } \ - if (_count != (head)->hh.tbl->num_items) { \ - HASH_OOPS("%s: invalid hh item count %u, actual %u\n", \ - (where), (head)->hh.tbl->num_items, _count); \ - } \ - _count = 0; \ - _prev = NULL; \ - _thh = &(head)->hh; \ - while (_thh) { \ - _count++; \ - if (_prev != (char*)_thh->prev) { \ - HASH_OOPS("%s: invalid prev %p, actual %p\n", \ - (where), (void*)_thh->prev, (void*)_prev); \ - } \ - _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \ - _thh = (_thh->next ? HH_FROM_ELMT((head)->hh.tbl, _thh->next) : NULL); \ - } \ - if (_count != (head)->hh.tbl->num_items) { \ - HASH_OOPS("%s: invalid app item count %u, actual %u\n", \ - (where), (head)->hh.tbl->num_items, _count); \ - } \ - } \ -} while (0) -#else -#define HASH_FSCK(hh,head,where) -#endif - -/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to - * the descriptor to which this macro is defined for tuning the hash function. - * The app can #include to get the prototype for write(2). */ -#ifdef HASH_EMIT_KEYS -#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \ -do { \ - unsigned _klen = fieldlen; \ - write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \ - write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ -} while (0) -#else -#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) -#endif - -/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */ -#ifdef HASH_FUNCTION -#define HASH_FCN HASH_FUNCTION -#else -#define HASH_FCN HASH_JEN -#endif - -/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */ -#define HASH_BER(key,keylen,hashv) \ -do { \ - unsigned _hb_keylen = (unsigned)keylen; \ - const unsigned char *_hb_key = (const unsigned char*)(key); \ - (hashv) = 0; \ - while (_hb_keylen-- != 0U) { \ - (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++; \ - } \ -} while (0) - - -/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at - * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */ -#define HASH_SAX(key,keylen,hashv) \ -do { \ - unsigned _sx_i; \ - const unsigned char *_hs_key = (const unsigned char*)(key); \ - hashv = 0; \ - for (_sx_i=0; _sx_i < keylen; _sx_i++) { \ - hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \ - } \ -} while (0) -/* FNV-1a variation */ -#define HASH_FNV(key,keylen,hashv) \ -do { \ - unsigned _fn_i; \ - const unsigned char *_hf_key = (const unsigned char*)(key); \ - (hashv) = 2166136261U; \ - for (_fn_i=0; _fn_i < keylen; _fn_i++) { \ - hashv = hashv ^ _hf_key[_fn_i]; \ - hashv = hashv * 16777619U; \ - } \ -} while (0) - -#define HASH_OAT(key,keylen,hashv) \ -do { \ - unsigned _ho_i; \ - const unsigned char *_ho_key=(const unsigned char*)(key); \ - hashv = 0; \ - for(_ho_i=0; _ho_i < keylen; _ho_i++) { \ - hashv += _ho_key[_ho_i]; \ - hashv += (hashv << 10); \ - hashv ^= (hashv >> 6); \ - } \ - hashv += (hashv << 3); \ - hashv ^= (hashv >> 11); \ - hashv += (hashv << 15); \ -} while (0) - -#define HASH_JEN_MIX(a,b,c) \ -do { \ - a -= b; a -= c; a ^= ( c >> 13 ); \ - b -= c; b -= a; b ^= ( a << 8 ); \ - c -= a; c -= b; c ^= ( b >> 13 ); \ - a -= b; a -= c; a ^= ( c >> 12 ); \ - b -= c; b -= a; b ^= ( a << 16 ); \ - c -= a; c -= b; c ^= ( b >> 5 ); \ - a -= b; a -= c; a ^= ( c >> 3 ); \ - b -= c; b -= a; b ^= ( a << 10 ); \ - c -= a; c -= b; c ^= ( b >> 15 ); \ -} while (0) - -#define HASH_JEN(key,keylen,hashv) \ -do { \ - unsigned _hj_i,_hj_j,_hj_k; \ - unsigned const char *_hj_key=(unsigned const char*)(key); \ - hashv = 0xfeedbeefu; \ - _hj_i = _hj_j = 0x9e3779b9u; \ - _hj_k = (unsigned)(keylen); \ - while (_hj_k >= 12U) { \ - _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \ - + ( (unsigned)_hj_key[2] << 16 ) \ - + ( (unsigned)_hj_key[3] << 24 ) ); \ - _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \ - + ( (unsigned)_hj_key[6] << 16 ) \ - + ( (unsigned)_hj_key[7] << 24 ) ); \ - hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \ - + ( (unsigned)_hj_key[10] << 16 ) \ - + ( (unsigned)_hj_key[11] << 24 ) ); \ - \ - HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ - \ - _hj_key += 12; \ - _hj_k -= 12U; \ - } \ - hashv += (unsigned)(keylen); \ - switch ( _hj_k ) { \ - case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); /* FALLTHROUGH */ \ - case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); /* FALLTHROUGH */ \ - case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); /* FALLTHROUGH */ \ - case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); /* FALLTHROUGH */ \ - case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); /* FALLTHROUGH */ \ - case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); /* FALLTHROUGH */ \ - case 5: _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ - case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); /* FALLTHROUGH */ \ - case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); /* FALLTHROUGH */ \ - case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); /* FALLTHROUGH */ \ - case 1: _hj_i += _hj_key[0]; \ - } \ - HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ -} while (0) - -/* The Paul Hsieh hash function */ -#undef get16bits -#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ - || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) -#define get16bits(d) (*((const uint16_t *) (d))) -#endif - -#if !defined (get16bits) -#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \ - +(uint32_t)(((const uint8_t *)(d))[0]) ) -#endif -#define HASH_SFH(key,keylen,hashv) \ -do { \ - unsigned const char *_sfh_key=(unsigned const char*)(key); \ - uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen; \ - \ - unsigned _sfh_rem = _sfh_len & 3U; \ - _sfh_len >>= 2; \ - hashv = 0xcafebabeu; \ - \ - /* Main loop */ \ - for (;_sfh_len > 0U; _sfh_len--) { \ - hashv += get16bits (_sfh_key); \ - _sfh_tmp = ((uint32_t)(get16bits (_sfh_key+2)) << 11) ^ hashv; \ - hashv = (hashv << 16) ^ _sfh_tmp; \ - _sfh_key += 2U*sizeof (uint16_t); \ - hashv += hashv >> 11; \ - } \ - \ - /* Handle end cases */ \ - switch (_sfh_rem) { \ - case 3: hashv += get16bits (_sfh_key); \ - hashv ^= hashv << 16; \ - hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)]) << 18; \ - hashv += hashv >> 11; \ - break; \ - case 2: hashv += get16bits (_sfh_key); \ - hashv ^= hashv << 11; \ - hashv += hashv >> 17; \ - break; \ - case 1: hashv += *_sfh_key; \ - hashv ^= hashv << 10; \ - hashv += hashv >> 1; \ - } \ - \ - /* Force "avalanching" of final 127 bits */ \ - hashv ^= hashv << 3; \ - hashv += hashv >> 5; \ - hashv ^= hashv << 4; \ - hashv += hashv >> 17; \ - hashv ^= hashv << 25; \ - hashv += hashv >> 6; \ -} while (0) - -/* iterate over items in a known bucket to find desired item */ -#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,hashval,out) \ -do { \ - if ((head).hh_head != NULL) { \ - DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head)); \ - } else { \ - (out) = NULL; \ - } \ - while ((out) != NULL) { \ - if ((out)->hh.hashv == (hashval) && (out)->hh.keylen == (keylen_in)) { \ - if (HASH_KEYCMP((out)->hh.key, keyptr, keylen_in) == 0) { \ - break; \ - } \ - } \ - if ((out)->hh.hh_next != NULL) { \ - DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next)); \ - } else { \ - (out) = NULL; \ - } \ - } \ -} while (0) - -/* add an item to a bucket */ -#define HASH_ADD_TO_BKT(head,hh,addhh,oomed) \ -do { \ - UT_hash_bucket *_ha_head = &(head); \ - _ha_head->count++; \ - (addhh)->hh_next = _ha_head->hh_head; \ - (addhh)->hh_prev = NULL; \ - if (_ha_head->hh_head != NULL) { \ - _ha_head->hh_head->hh_prev = (addhh); \ - } \ - _ha_head->hh_head = (addhh); \ - if ((_ha_head->count >= ((_ha_head->expand_mult + 1U) * HASH_BKT_CAPACITY_THRESH)) \ - && !(addhh)->tbl->noexpand) { \ - HASH_EXPAND_BUCKETS(addhh,(addhh)->tbl, oomed); \ - IF_HASH_NONFATAL_OOM( \ - if (oomed) { \ - HASH_DEL_IN_BKT(head,addhh); \ - } \ - ) \ - } \ -} while (0) - -/* remove an item from a given bucket */ -#define HASH_DEL_IN_BKT(head,delhh) \ -do { \ - UT_hash_bucket *_hd_head = &(head); \ - _hd_head->count--; \ - if (_hd_head->hh_head == (delhh)) { \ - _hd_head->hh_head = (delhh)->hh_next; \ - } \ - if ((delhh)->hh_prev) { \ - (delhh)->hh_prev->hh_next = (delhh)->hh_next; \ - } \ - if ((delhh)->hh_next) { \ - (delhh)->hh_next->hh_prev = (delhh)->hh_prev; \ - } \ -} while (0) - -/* Bucket expansion has the effect of doubling the number of buckets - * and redistributing the items into the new buckets. Ideally the - * items will distribute more or less evenly into the new buckets - * (the extent to which this is true is a measure of the quality of - * the hash function as it applies to the key domain). - * - * With the items distributed into more buckets, the chain length - * (item count) in each bucket is reduced. Thus by expanding buckets - * the hash keeps a bound on the chain length. This bounded chain - * length is the essence of how a hash provides constant time lookup. - * - * The calculation of tbl->ideal_chain_maxlen below deserves some - * explanation. First, keep in mind that we're calculating the ideal - * maximum chain length based on the *new* (doubled) bucket count. - * In fractions this is just n/b (n=number of items,b=new num buckets). - * Since the ideal chain length is an integer, we want to calculate - * ceil(n/b). We don't depend on floating point arithmetic in this - * hash, so to calculate ceil(n/b) with integers we could write - * - * ceil(n/b) = (n/b) + ((n%b)?1:0) - * - * and in fact a previous version of this hash did just that. - * But now we have improved things a bit by recognizing that b is - * always a power of two. We keep its base 2 log handy (call it lb), - * so now we can write this with a bit shift and logical AND: - * - * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0) - * - */ -#define HASH_EXPAND_BUCKETS(hh,tbl,oomed) \ -do { \ - unsigned _he_bkt; \ - unsigned _he_bkt_i; \ - struct UT_hash_handle *_he_thh, *_he_hh_nxt; \ - UT_hash_bucket *_he_new_buckets, *_he_newbkt; \ - _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \ - 2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ - if (!_he_new_buckets) { \ - HASH_RECORD_OOM(oomed); \ - } else { \ - uthash_bzero(_he_new_buckets, \ - 2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ - (tbl)->ideal_chain_maxlen = \ - ((tbl)->num_items >> ((tbl)->log2_num_buckets+1U)) + \ - ((((tbl)->num_items & (((tbl)->num_buckets*2U)-1U)) != 0U) ? 1U : 0U); \ - (tbl)->nonideal_items = 0; \ - for (_he_bkt_i = 0; _he_bkt_i < (tbl)->num_buckets; _he_bkt_i++) { \ - _he_thh = (tbl)->buckets[ _he_bkt_i ].hh_head; \ - while (_he_thh != NULL) { \ - _he_hh_nxt = _he_thh->hh_next; \ - HASH_TO_BKT(_he_thh->hashv, (tbl)->num_buckets * 2U, _he_bkt); \ - _he_newbkt = &(_he_new_buckets[_he_bkt]); \ - if (++(_he_newbkt->count) > (tbl)->ideal_chain_maxlen) { \ - (tbl)->nonideal_items++; \ - if (_he_newbkt->count > _he_newbkt->expand_mult * (tbl)->ideal_chain_maxlen) { \ - _he_newbkt->expand_mult++; \ - } \ - } \ - _he_thh->hh_prev = NULL; \ - _he_thh->hh_next = _he_newbkt->hh_head; \ - if (_he_newbkt->hh_head != NULL) { \ - _he_newbkt->hh_head->hh_prev = _he_thh; \ - } \ - _he_newbkt->hh_head = _he_thh; \ - _he_thh = _he_hh_nxt; \ - } \ - } \ - uthash_free((tbl)->buckets, (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ - (tbl)->num_buckets *= 2U; \ - (tbl)->log2_num_buckets++; \ - (tbl)->buckets = _he_new_buckets; \ - (tbl)->ineff_expands = ((tbl)->nonideal_items > ((tbl)->num_items >> 1)) ? \ - ((tbl)->ineff_expands+1U) : 0U; \ - if ((tbl)->ineff_expands > 1U) { \ - (tbl)->noexpand = 1; \ - uthash_noexpand_fyi(tbl); \ - } \ - uthash_expand_fyi(tbl); \ - } \ -} while (0) - - -/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */ -/* Note that HASH_SORT assumes the hash handle name to be hh. - * HASH_SRT was added to allow the hash handle name to be passed in. */ -#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn) -#define HASH_SRT(hh,head,cmpfcn) \ -do { \ - unsigned _hs_i; \ - unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \ - struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \ - if (head != NULL) { \ - _hs_insize = 1; \ - _hs_looping = 1; \ - _hs_list = &((head)->hh); \ - while (_hs_looping != 0U) { \ - _hs_p = _hs_list; \ - _hs_list = NULL; \ - _hs_tail = NULL; \ - _hs_nmerges = 0; \ - while (_hs_p != NULL) { \ - _hs_nmerges++; \ - _hs_q = _hs_p; \ - _hs_psize = 0; \ - for (_hs_i = 0; _hs_i < _hs_insize; ++_hs_i) { \ - _hs_psize++; \ - _hs_q = ((_hs_q->next != NULL) ? \ - HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ - if (_hs_q == NULL) { \ - break; \ - } \ - } \ - _hs_qsize = _hs_insize; \ - while ((_hs_psize != 0U) || ((_hs_qsize != 0U) && (_hs_q != NULL))) { \ - if (_hs_psize == 0U) { \ - _hs_e = _hs_q; \ - _hs_q = ((_hs_q->next != NULL) ? \ - HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ - _hs_qsize--; \ - } else if ((_hs_qsize == 0U) || (_hs_q == NULL)) { \ - _hs_e = _hs_p; \ - if (_hs_p != NULL) { \ - _hs_p = ((_hs_p->next != NULL) ? \ - HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL); \ - } \ - _hs_psize--; \ - } else if ((cmpfcn( \ - DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_p)), \ - DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_q)) \ - )) <= 0) { \ - _hs_e = _hs_p; \ - if (_hs_p != NULL) { \ - _hs_p = ((_hs_p->next != NULL) ? \ - HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL); \ - } \ - _hs_psize--; \ - } else { \ - _hs_e = _hs_q; \ - _hs_q = ((_hs_q->next != NULL) ? \ - HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ - _hs_qsize--; \ - } \ - if ( _hs_tail != NULL ) { \ - _hs_tail->next = ((_hs_e != NULL) ? \ - ELMT_FROM_HH((head)->hh.tbl, _hs_e) : NULL); \ - } else { \ - _hs_list = _hs_e; \ - } \ - if (_hs_e != NULL) { \ - _hs_e->prev = ((_hs_tail != NULL) ? \ - ELMT_FROM_HH((head)->hh.tbl, _hs_tail) : NULL); \ - } \ - _hs_tail = _hs_e; \ - } \ - _hs_p = _hs_q; \ - } \ - if (_hs_tail != NULL) { \ - _hs_tail->next = NULL; \ - } \ - if (_hs_nmerges <= 1U) { \ - _hs_looping = 0; \ - (head)->hh.tbl->tail = _hs_tail; \ - DECLTYPE_ASSIGN(head, ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \ - } \ - _hs_insize *= 2U; \ - } \ - HASH_FSCK(hh, head, "HASH_SRT"); \ - } \ -} while (0) - -/* This function selects items from one hash into another hash. - * The end result is that the selected items have dual presence - * in both hashes. There is no copy of the items made; rather - * they are added into the new hash through a secondary hash - * hash handle that must be present in the structure. */ -#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \ -do { \ - unsigned _src_bkt, _dst_bkt; \ - void *_last_elt = NULL, *_elt; \ - UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \ - ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \ - if ((src) != NULL) { \ - for (_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \ - for (_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \ - _src_hh != NULL; \ - _src_hh = _src_hh->hh_next) { \ - _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \ - if (cond(_elt)) { \ - IF_HASH_NONFATAL_OOM( int _hs_oomed = 0; ) \ - _dst_hh = (UT_hash_handle*)(void*)(((char*)_elt) + _dst_hho); \ - _dst_hh->key = _src_hh->key; \ - _dst_hh->keylen = _src_hh->keylen; \ - _dst_hh->hashv = _src_hh->hashv; \ - _dst_hh->prev = _last_elt; \ - _dst_hh->next = NULL; \ - if (_last_elt_hh != NULL) { \ - _last_elt_hh->next = _elt; \ - } \ - if ((dst) == NULL) { \ - DECLTYPE_ASSIGN(dst, _elt); \ - HASH_MAKE_TABLE(hh_dst, dst, _hs_oomed); \ - IF_HASH_NONFATAL_OOM( \ - if (_hs_oomed) { \ - uthash_nonfatal_oom(_elt); \ - (dst) = NULL; \ - continue; \ - } \ - ) \ - } else { \ - _dst_hh->tbl = (dst)->hh_dst.tbl; \ - } \ - HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \ - HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt], hh_dst, _dst_hh, _hs_oomed); \ - (dst)->hh_dst.tbl->num_items++; \ - IF_HASH_NONFATAL_OOM( \ - if (_hs_oomed) { \ - HASH_ROLLBACK_BKT(hh_dst, dst, _dst_hh); \ - HASH_DELETE_HH(hh_dst, dst, _dst_hh); \ - _dst_hh->tbl = NULL; \ - uthash_nonfatal_oom(_elt); \ - continue; \ - } \ - ) \ - HASH_BLOOM_ADD(_dst_hh->tbl, _dst_hh->hashv); \ - _last_elt = _elt; \ - _last_elt_hh = _dst_hh; \ - } \ - } \ - } \ - } \ - HASH_FSCK(hh_dst, dst, "HASH_SELECT"); \ -} while (0) - -#define HASH_CLEAR(hh,head) \ -do { \ - if ((head) != NULL) { \ - HASH_BLOOM_FREE((head)->hh.tbl); \ - uthash_free((head)->hh.tbl->buckets, \ - (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \ - uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ - (head) = NULL; \ - } \ -} while (0) - -#define HASH_OVERHEAD(hh,head) \ - (((head) != NULL) ? ( \ - (size_t)(((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \ - ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \ - sizeof(UT_hash_table) + \ - (HASH_BLOOM_BYTELEN))) : 0U) - -#ifdef NO_DECLTYPE -#define HASH_ITER(hh,head,el,tmp) \ -for(((el)=(head)), ((*(char**)(&(tmp)))=(char*)((head!=NULL)?(head)->hh.next:NULL)); \ - (el) != NULL; ((el)=(tmp)), ((*(char**)(&(tmp)))=(char*)((tmp!=NULL)?(tmp)->hh.next:NULL))) -#else -#define HASH_ITER(hh,head,el,tmp) \ -for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL)); \ - (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL))) -#endif - -/* obtain a count of items in the hash */ -#define HASH_COUNT(head) HASH_CNT(hh,head) -#define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U) - -typedef struct UT_hash_bucket { - struct UT_hash_handle *hh_head; - unsigned count; - - /* expand_mult is normally set to 0. In this situation, the max chain length - * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If - * the bucket's chain exceeds this length, bucket expansion is triggered). - * However, setting expand_mult to a non-zero value delays bucket expansion - * (that would be triggered by additions to this particular bucket) - * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH. - * (The multiplier is simply expand_mult+1). The whole idea of this - * multiplier is to reduce bucket expansions, since they are expensive, in - * situations where we know that a particular bucket tends to be overused. - * It is better to let its chain length grow to a longer yet-still-bounded - * value, than to do an O(n) bucket expansion too often. - */ - unsigned expand_mult; - -} UT_hash_bucket; - -/* random signature used only to find hash tables in external analysis */ -#define HASH_SIGNATURE 0xa0111fe1u -#define HASH_BLOOM_SIGNATURE 0xb12220f2u - -typedef struct UT_hash_table { - UT_hash_bucket *buckets; - unsigned num_buckets, log2_num_buckets; - unsigned num_items; - struct UT_hash_handle *tail; /* tail hh in app order, for fast append */ - ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */ - - /* in an ideal situation (all buckets used equally), no bucket would have - * more than ceil(#items/#buckets) items. that's the ideal chain length. */ - unsigned ideal_chain_maxlen; - - /* nonideal_items is the number of items in the hash whose chain position - * exceeds the ideal chain maxlen. these items pay the penalty for an uneven - * hash distribution; reaching them in a chain traversal takes >ideal steps */ - unsigned nonideal_items; - - /* ineffective expands occur when a bucket doubling was performed, but - * afterward, more than half the items in the hash had nonideal chain - * positions. If this happens on two consecutive expansions we inhibit any - * further expansion, as it's not helping; this happens when the hash - * function isn't a good fit for the key domain. When expansion is inhibited - * the hash will still work, albeit no longer in constant time. */ - unsigned ineff_expands, noexpand; - - uint32_t signature; /* used only to find hash tables in external analysis */ -#ifdef HASH_BLOOM - uint32_t bloom_sig; /* used only to test bloom exists in external analysis */ - uint8_t *bloom_bv; - uint8_t bloom_nbits; -#endif - -} UT_hash_table; - -typedef struct UT_hash_handle { - struct UT_hash_table *tbl; - void *prev; /* prev element in app order */ - void *next; /* next element in app order */ - struct UT_hash_handle *hh_prev; /* previous hh in bucket order */ - struct UT_hash_handle *hh_next; /* next hh in bucket order */ - void *key; /* ptr to enclosing struct's key */ - unsigned keylen; /* enclosing struct's key len */ - unsigned hashv; /* result of hash-fcn(key) */ -} UT_hash_handle; - -#endif /* UTHASH_H */ diff --git a/src/inc_internal/uthash/utlist.h b/src/inc_internal/uthash/utlist.h deleted file mode 100644 index 5bb1ac9..0000000 --- a/src/inc_internal/uthash/utlist.h +++ /dev/null @@ -1,1073 +0,0 @@ -/* -Copyright (c) 2007-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef UTLIST_H -#define UTLIST_H - -#define UTLIST_VERSION 2.1.0 - -#include - -/* - * This file contains macros to manipulate singly and doubly-linked lists. - * - * 1. LL_ macros: singly-linked lists. - * 2. DL_ macros: doubly-linked lists. - * 3. CDL_ macros: circular doubly-linked lists. - * - * To use singly-linked lists, your structure must have a "next" pointer. - * To use doubly-linked lists, your structure must "prev" and "next" pointers. - * Either way, the pointer to the head of the list must be initialized to NULL. - * - * ----------------.EXAMPLE ------------------------- - * struct item { - * int id; - * struct item *prev, *next; - * } - * - * struct item *list = NULL: - * - * int main() { - * struct item *item; - * ... allocate and populate item ... - * DL_APPEND(list, item); - * } - * -------------------------------------------------- - * - * For doubly-linked lists, the append and delete macros are O(1) - * For singly-linked lists, append and delete are O(n) but prepend is O(1) - * The sort macro is O(n log(n)) for all types of single/double/circular lists. - */ - -/* These macros use decltype or the earlier __typeof GNU extension. - As decltype is only available in newer compilers (VS2010 or gcc 4.3+ - when compiling c++ source) this code uses whatever method is needed - or, for VS2008 where neither is available, uses casting workarounds. */ -#if !defined(LDECLTYPE) && !defined(NO_DECLTYPE) -#if defined(_MSC_VER) /* MS compiler */ -#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ -#define LDECLTYPE(x) decltype(x) -#else /* VS2008 or older (or VS2010 in C mode) */ -#define NO_DECLTYPE -#endif -#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__) -#define NO_DECLTYPE -#else /* GNU, Sun and other compilers */ -#define LDECLTYPE(x) __typeof(x) -#endif -#endif - -/* for VS2008 we use some workarounds to get around the lack of decltype, - * namely, we always reassign our tmp variable to the list head if we need - * to dereference its prev/next pointers, and save/restore the real head.*/ -#ifdef NO_DECLTYPE -#define IF_NO_DECLTYPE(x) x -#define LDECLTYPE(x) char* -#define UTLIST_SV(elt,list) _tmp = (char*)(list); {char **_alias = (char**)&(list); *_alias = (elt); } -#define UTLIST_NEXT(elt,list,next) ((char*)((list)->next)) -#define UTLIST_NEXTASGN(elt,list,to,next) { char **_alias = (char**)&((list)->next); *_alias=(char*)(to); } -/* #define UTLIST_PREV(elt,list,prev) ((char*)((list)->prev)) */ -#define UTLIST_PREVASGN(elt,list,to,prev) { char **_alias = (char**)&((list)->prev); *_alias=(char*)(to); } -#define UTLIST_RS(list) { char **_alias = (char**)&(list); *_alias=_tmp; } -#define UTLIST_CASTASGN(a,b) { char **_alias = (char**)&(a); *_alias=(char*)(b); } -#else -#define IF_NO_DECLTYPE(x) -#define UTLIST_SV(elt,list) -#define UTLIST_NEXT(elt,list,next) ((elt)->next) -#define UTLIST_NEXTASGN(elt,list,to,next) ((elt)->next)=(to) -/* #define UTLIST_PREV(elt,list,prev) ((elt)->prev) */ -#define UTLIST_PREVASGN(elt,list,to,prev) ((elt)->prev)=(to) -#define UTLIST_RS(list) -#define UTLIST_CASTASGN(a,b) (a)=(b) -#endif - -/****************************************************************************** - * The sort macro is an adaptation of Simon Tatham's O(n log(n)) mergesort * - * Unwieldy variable names used here to avoid shadowing passed-in variables. * - *****************************************************************************/ -#define LL_SORT(list, cmp) \ - LL_SORT2(list, cmp, next) - -#define LL_SORT2(list, cmp, next) \ -do { \ - LDECLTYPE(list) _ls_p; \ - LDECLTYPE(list) _ls_q; \ - LDECLTYPE(list) _ls_e; \ - LDECLTYPE(list) _ls_tail; \ - IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ - int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ - if (list) { \ - _ls_insize = 1; \ - _ls_looping = 1; \ - while (_ls_looping) { \ - UTLIST_CASTASGN(_ls_p,list); \ - (list) = NULL; \ - _ls_tail = NULL; \ - _ls_nmerges = 0; \ - while (_ls_p) { \ - _ls_nmerges++; \ - _ls_q = _ls_p; \ - _ls_psize = 0; \ - for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ - _ls_psize++; \ - UTLIST_SV(_ls_q,list); _ls_q = UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); \ - if (!_ls_q) break; \ - } \ - _ls_qsize = _ls_insize; \ - while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ - if (_ls_psize == 0) { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - } else if (_ls_qsize == 0 || !_ls_q) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - } else if (cmp(_ls_p,_ls_q) <= 0) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - } else { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - } \ - if (_ls_tail) { \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,_ls_e,next); UTLIST_RS(list); \ - } else { \ - UTLIST_CASTASGN(list,_ls_e); \ - } \ - _ls_tail = _ls_e; \ - } \ - _ls_p = _ls_q; \ - } \ - if (_ls_tail) { \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,NULL,next); UTLIST_RS(list); \ - } \ - if (_ls_nmerges <= 1) { \ - _ls_looping=0; \ - } \ - _ls_insize *= 2; \ - } \ - } \ -} while (0) - - -#define DL_SORT(list, cmp) \ - DL_SORT2(list, cmp, prev, next) - -#define DL_SORT2(list, cmp, prev, next) \ -do { \ - LDECLTYPE(list) _ls_p; \ - LDECLTYPE(list) _ls_q; \ - LDECLTYPE(list) _ls_e; \ - LDECLTYPE(list) _ls_tail; \ - IF_NO_DECLTYPE(LDECLTYPE(list) _tmp;) \ - int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ - if (list) { \ - _ls_insize = 1; \ - _ls_looping = 1; \ - while (_ls_looping) { \ - UTLIST_CASTASGN(_ls_p,list); \ - (list) = NULL; \ - _ls_tail = NULL; \ - _ls_nmerges = 0; \ - while (_ls_p) { \ - _ls_nmerges++; \ - _ls_q = _ls_p; \ - _ls_psize = 0; \ - for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ - _ls_psize++; \ - UTLIST_SV(_ls_q,list); _ls_q = UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); \ - if (!_ls_q) break; \ - } \ - _ls_qsize = _ls_insize; \ - while ((_ls_psize > 0) || ((_ls_qsize > 0) && _ls_q)) { \ - if (_ls_psize == 0) { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - } else if ((_ls_qsize == 0) || (!_ls_q)) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - } else if (cmp(_ls_p,_ls_q) <= 0) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - } else { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - } \ - if (_ls_tail) { \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,_ls_e,next); UTLIST_RS(list); \ - } else { \ - UTLIST_CASTASGN(list,_ls_e); \ - } \ - UTLIST_SV(_ls_e,list); UTLIST_PREVASGN(_ls_e,list,_ls_tail,prev); UTLIST_RS(list); \ - _ls_tail = _ls_e; \ - } \ - _ls_p = _ls_q; \ - } \ - UTLIST_CASTASGN((list)->prev, _ls_tail); \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,NULL,next); UTLIST_RS(list); \ - if (_ls_nmerges <= 1) { \ - _ls_looping=0; \ - } \ - _ls_insize *= 2; \ - } \ - } \ -} while (0) - -#define CDL_SORT(list, cmp) \ - CDL_SORT2(list, cmp, prev, next) - -#define CDL_SORT2(list, cmp, prev, next) \ -do { \ - LDECLTYPE(list) _ls_p; \ - LDECLTYPE(list) _ls_q; \ - LDECLTYPE(list) _ls_e; \ - LDECLTYPE(list) _ls_tail; \ - LDECLTYPE(list) _ls_oldhead; \ - LDECLTYPE(list) _tmp; \ - int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping; \ - if (list) { \ - _ls_insize = 1; \ - _ls_looping = 1; \ - while (_ls_looping) { \ - UTLIST_CASTASGN(_ls_p,list); \ - UTLIST_CASTASGN(_ls_oldhead,list); \ - (list) = NULL; \ - _ls_tail = NULL; \ - _ls_nmerges = 0; \ - while (_ls_p) { \ - _ls_nmerges++; \ - _ls_q = _ls_p; \ - _ls_psize = 0; \ - for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) { \ - _ls_psize++; \ - UTLIST_SV(_ls_q,list); \ - if (UTLIST_NEXT(_ls_q,list,next) == _ls_oldhead) { \ - _ls_q = NULL; \ - } else { \ - _ls_q = UTLIST_NEXT(_ls_q,list,next); \ - } \ - UTLIST_RS(list); \ - if (!_ls_q) break; \ - } \ - _ls_qsize = _ls_insize; \ - while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) { \ - if (_ls_psize == 0) { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ - } else if (_ls_qsize == 0 || !_ls_q) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ - } else if (cmp(_ls_p,_ls_q) <= 0) { \ - _ls_e = _ls_p; UTLIST_SV(_ls_p,list); _ls_p = \ - UTLIST_NEXT(_ls_p,list,next); UTLIST_RS(list); _ls_psize--; \ - if (_ls_p == _ls_oldhead) { _ls_p = NULL; } \ - } else { \ - _ls_e = _ls_q; UTLIST_SV(_ls_q,list); _ls_q = \ - UTLIST_NEXT(_ls_q,list,next); UTLIST_RS(list); _ls_qsize--; \ - if (_ls_q == _ls_oldhead) { _ls_q = NULL; } \ - } \ - if (_ls_tail) { \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,_ls_e,next); UTLIST_RS(list); \ - } else { \ - UTLIST_CASTASGN(list,_ls_e); \ - } \ - UTLIST_SV(_ls_e,list); UTLIST_PREVASGN(_ls_e,list,_ls_tail,prev); UTLIST_RS(list); \ - _ls_tail = _ls_e; \ - } \ - _ls_p = _ls_q; \ - } \ - UTLIST_CASTASGN((list)->prev,_ls_tail); \ - UTLIST_CASTASGN(_tmp,list); \ - UTLIST_SV(_ls_tail,list); UTLIST_NEXTASGN(_ls_tail,list,_tmp,next); UTLIST_RS(list); \ - if (_ls_nmerges <= 1) { \ - _ls_looping=0; \ - } \ - _ls_insize *= 2; \ - } \ - } \ -} while (0) - -/****************************************************************************** - * singly linked list macros (non-circular) * - *****************************************************************************/ -#define LL_PREPEND(head,add) \ - LL_PREPEND2(head,add,next) - -#define LL_PREPEND2(head,add,next) \ -do { \ - (add)->next = (head); \ - (head) = (add); \ -} while (0) - -#define LL_CONCAT(head1,head2) \ - LL_CONCAT2(head1,head2,next) - -#define LL_CONCAT2(head1,head2,next) \ -do { \ - LDECLTYPE(head1) _tmp; \ - if (head1) { \ - _tmp = (head1); \ - while (_tmp->next) { _tmp = _tmp->next; } \ - _tmp->next=(head2); \ - } else { \ - (head1)=(head2); \ - } \ -} while (0) - -#define LL_APPEND(head,add) \ - LL_APPEND2(head,add,next) - -#define LL_APPEND2(head,add,next) \ -do { \ - LDECLTYPE(head) _tmp; \ - (add)->next=NULL; \ - if (head) { \ - _tmp = (head); \ - while (_tmp->next) { _tmp = _tmp->next; } \ - _tmp->next=(add); \ - } else { \ - (head)=(add); \ - } \ -} while (0) - -#define LL_INSERT_INORDER(head,add,cmp) \ - LL_INSERT_INORDER2(head,add,cmp,next) - -#define LL_INSERT_INORDER2(head,add,cmp,next) \ -do { \ - LDECLTYPE(head) _tmp; \ - if (head) { \ - LL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ - LL_APPEND_ELEM2(head, _tmp, add, next); \ - } else { \ - (head) = (add); \ - (head)->next = NULL; \ - } \ -} while (0) - -#define LL_LOWER_BOUND(head,elt,like,cmp) \ - LL_LOWER_BOUND2(head,elt,like,cmp,next) - -#define LL_LOWER_BOUND2(head,elt,like,cmp,next) \ - do { \ - if ((head) == NULL || (cmp(head, like)) >= 0) { \ - (elt) = NULL; \ - } else { \ - for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ - if (cmp((elt)->next, like) >= 0) { \ - break; \ - } \ - } \ - } \ - } while (0) - -#define LL_DELETE(head,del) \ - LL_DELETE2(head,del,next) - -#define LL_DELETE2(head,del,next) \ -do { \ - LDECLTYPE(head) _tmp; \ - if ((head) == (del)) { \ - (head)=(head)->next; \ - } else { \ - _tmp = (head); \ - while (_tmp->next && (_tmp->next != (del))) { \ - _tmp = _tmp->next; \ - } \ - if (_tmp->next) { \ - _tmp->next = (del)->next; \ - } \ - } \ -} while (0) - -#define LL_COUNT(head,el,counter) \ - LL_COUNT2(head,el,counter,next) \ - -#define LL_COUNT2(head,el,counter,next) \ -do { \ - (counter) = 0; \ - LL_FOREACH2(head,el,next) { ++(counter); } \ -} while (0) - -#define LL_FOREACH(head,el) \ - LL_FOREACH2(head,el,next) - -#define LL_FOREACH2(head,el,next) \ - for ((el) = (head); el; (el) = (el)->next) - -#define LL_FOREACH_SAFE(head,el,tmp) \ - LL_FOREACH_SAFE2(head,el,tmp,next) - -#define LL_FOREACH_SAFE2(head,el,tmp,next) \ - for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) - -#define LL_SEARCH_SCALAR(head,out,field,val) \ - LL_SEARCH_SCALAR2(head,out,field,val,next) - -#define LL_SEARCH_SCALAR2(head,out,field,val,next) \ -do { \ - LL_FOREACH2(head,out,next) { \ - if ((out)->field == (val)) break; \ - } \ -} while (0) - -#define LL_SEARCH(head,out,elt,cmp) \ - LL_SEARCH2(head,out,elt,cmp,next) - -#define LL_SEARCH2(head,out,elt,cmp,next) \ -do { \ - LL_FOREACH2(head,out,next) { \ - if ((cmp(out,elt))==0) break; \ - } \ -} while (0) - -#define LL_REPLACE_ELEM2(head, el, add, next) \ -do { \ - LDECLTYPE(head) _tmp; \ - assert((head) != NULL); \ - assert((el) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el)->next; \ - if ((head) == (el)) { \ - (head) = (add); \ - } else { \ - _tmp = (head); \ - while (_tmp->next && (_tmp->next != (el))) { \ - _tmp = _tmp->next; \ - } \ - if (_tmp->next) { \ - _tmp->next = (add); \ - } \ - } \ -} while (0) - -#define LL_REPLACE_ELEM(head, el, add) \ - LL_REPLACE_ELEM2(head, el, add, next) - -#define LL_PREPEND_ELEM2(head, el, add, next) \ -do { \ - if (el) { \ - LDECLTYPE(head) _tmp; \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el); \ - if ((head) == (el)) { \ - (head) = (add); \ - } else { \ - _tmp = (head); \ - while (_tmp->next && (_tmp->next != (el))) { \ - _tmp = _tmp->next; \ - } \ - if (_tmp->next) { \ - _tmp->next = (add); \ - } \ - } \ - } else { \ - LL_APPEND2(head, add, next); \ - } \ -} while (0) \ - -#define LL_PREPEND_ELEM(head, el, add) \ - LL_PREPEND_ELEM2(head, el, add, next) - -#define LL_APPEND_ELEM2(head, el, add, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el)->next; \ - (el)->next = (add); \ - } else { \ - LL_PREPEND2(head, add, next); \ - } \ -} while (0) \ - -#define LL_APPEND_ELEM(head, el, add) \ - LL_APPEND_ELEM2(head, el, add, next) - -#ifdef NO_DECLTYPE -/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ - -#undef LL_CONCAT2 -#define LL_CONCAT2(head1,head2,next) \ -do { \ - char *_tmp; \ - if (head1) { \ - _tmp = (char*)(head1); \ - while ((head1)->next) { (head1) = (head1)->next; } \ - (head1)->next = (head2); \ - UTLIST_RS(head1); \ - } else { \ - (head1)=(head2); \ - } \ -} while (0) - -#undef LL_APPEND2 -#define LL_APPEND2(head,add,next) \ -do { \ - if (head) { \ - (add)->next = head; /* use add->next as a temp variable */ \ - while ((add)->next->next) { (add)->next = (add)->next->next; } \ - (add)->next->next=(add); \ - } else { \ - (head)=(add); \ - } \ - (add)->next=NULL; \ -} while (0) - -#undef LL_INSERT_INORDER2 -#define LL_INSERT_INORDER2(head,add,cmp,next) \ -do { \ - if ((head) == NULL || (cmp(head, add)) >= 0) { \ - (add)->next = (head); \ - (head) = (add); \ - } else { \ - char *_tmp = (char*)(head); \ - while ((head)->next != NULL && (cmp((head)->next, add)) < 0) { \ - (head) = (head)->next; \ - } \ - (add)->next = (head)->next; \ - (head)->next = (add); \ - UTLIST_RS(head); \ - } \ -} while (0) - -#undef LL_DELETE2 -#define LL_DELETE2(head,del,next) \ -do { \ - if ((head) == (del)) { \ - (head)=(head)->next; \ - } else { \ - char *_tmp = (char*)(head); \ - while ((head)->next && ((head)->next != (del))) { \ - (head) = (head)->next; \ - } \ - if ((head)->next) { \ - (head)->next = ((del)->next); \ - } \ - UTLIST_RS(head); \ - } \ -} while (0) - -#undef LL_REPLACE_ELEM2 -#define LL_REPLACE_ELEM2(head, el, add, next) \ -do { \ - assert((head) != NULL); \ - assert((el) != NULL); \ - assert((add) != NULL); \ - if ((head) == (el)) { \ - (head) = (add); \ - } else { \ - (add)->next = head; \ - while ((add)->next->next && ((add)->next->next != (el))) { \ - (add)->next = (add)->next->next; \ - } \ - if ((add)->next->next) { \ - (add)->next->next = (add); \ - } \ - } \ - (add)->next = (el)->next; \ -} while (0) - -#undef LL_PREPEND_ELEM2 -#define LL_PREPEND_ELEM2(head, el, add, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - if ((head) == (el)) { \ - (head) = (add); \ - } else { \ - (add)->next = (head); \ - while ((add)->next->next && ((add)->next->next != (el))) { \ - (add)->next = (add)->next->next; \ - } \ - if ((add)->next->next) { \ - (add)->next->next = (add); \ - } \ - } \ - (add)->next = (el); \ - } else { \ - LL_APPEND2(head, add, next); \ - } \ -} while (0) \ - -#endif /* NO_DECLTYPE */ - -/****************************************************************************** - * doubly linked list macros (non-circular) * - *****************************************************************************/ -#define DL_PREPEND(head,add) \ - DL_PREPEND2(head,add,prev,next) - -#define DL_PREPEND2(head,add,prev,next) \ -do { \ - (add)->next = (head); \ - if (head) { \ - (add)->prev = (head)->prev; \ - (head)->prev = (add); \ - } else { \ - (add)->prev = (add); \ - } \ - (head) = (add); \ -} while (0) - -#define DL_APPEND(head,add) \ - DL_APPEND2(head,add,prev,next) - -#define DL_APPEND2(head,add,prev,next) \ -do { \ - if (head) { \ - (add)->prev = (head)->prev; \ - (head)->prev->next = (add); \ - (head)->prev = (add); \ - (add)->next = NULL; \ - } else { \ - (head)=(add); \ - (head)->prev = (head); \ - (head)->next = NULL; \ - } \ -} while (0) - -#define DL_INSERT_INORDER(head,add,cmp) \ - DL_INSERT_INORDER2(head,add,cmp,prev,next) - -#define DL_INSERT_INORDER2(head,add,cmp,prev,next) \ -do { \ - LDECLTYPE(head) _tmp; \ - if (head) { \ - DL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ - DL_APPEND_ELEM2(head, _tmp, add, prev, next); \ - } else { \ - (head) = (add); \ - (head)->prev = (head); \ - (head)->next = NULL; \ - } \ -} while (0) - -#define DL_LOWER_BOUND(head,elt,like,cmp) \ - DL_LOWER_BOUND2(head,elt,like,cmp,next) - -#define DL_LOWER_BOUND2(head,elt,like,cmp,next) \ -do { \ - if ((head) == NULL || (cmp(head, like)) >= 0) { \ - (elt) = NULL; \ - } else { \ - for ((elt) = (head); (elt)->next != NULL; (elt) = (elt)->next) { \ - if ((cmp((elt)->next, like)) >= 0) { \ - break; \ - } \ - } \ - } \ -} while (0) - -#define DL_CONCAT(head1,head2) \ - DL_CONCAT2(head1,head2,prev,next) - -#define DL_CONCAT2(head1,head2,prev,next) \ -do { \ - LDECLTYPE(head1) _tmp; \ - if (head2) { \ - if (head1) { \ - UTLIST_CASTASGN(_tmp, (head2)->prev); \ - (head2)->prev = (head1)->prev; \ - (head1)->prev->next = (head2); \ - UTLIST_CASTASGN((head1)->prev, _tmp); \ - } else { \ - (head1)=(head2); \ - } \ - } \ -} while (0) - -#define DL_DELETE(head,del) \ - DL_DELETE2(head,del,prev,next) - -#define DL_DELETE2(head,del,prev,next) \ -do { \ - assert((head) != NULL); \ - assert((del)->prev != NULL); \ - if ((del)->prev == (del)) { \ - (head)=NULL; \ - } else if ((del)==(head)) { \ - (del)->next->prev = (del)->prev; \ - (head) = (del)->next; \ - } else { \ - (del)->prev->next = (del)->next; \ - if ((del)->next) { \ - (del)->next->prev = (del)->prev; \ - } else { \ - (head)->prev = (del)->prev; \ - } \ - } \ -} while (0) - -#define DL_COUNT(head,el,counter) \ - DL_COUNT2(head,el,counter,next) \ - -#define DL_COUNT2(head,el,counter,next) \ -do { \ - (counter) = 0; \ - DL_FOREACH2(head,el,next) { ++(counter); } \ -} while (0) - -#define DL_FOREACH(head,el) \ - DL_FOREACH2(head,el,next) - -#define DL_FOREACH2(head,el,next) \ - for ((el) = (head); el; (el) = (el)->next) - -/* this version is safe for deleting the elements during iteration */ -#define DL_FOREACH_SAFE(head,el,tmp) \ - DL_FOREACH_SAFE2(head,el,tmp,next) - -#define DL_FOREACH_SAFE2(head,el,tmp,next) \ - for ((el) = (head); (el) && ((tmp) = (el)->next, 1); (el) = (tmp)) - -/* these are identical to their singly-linked list counterparts */ -#define DL_SEARCH_SCALAR LL_SEARCH_SCALAR -#define DL_SEARCH LL_SEARCH -#define DL_SEARCH_SCALAR2 LL_SEARCH_SCALAR2 -#define DL_SEARCH2 LL_SEARCH2 - -#define DL_REPLACE_ELEM2(head, el, add, prev, next) \ -do { \ - assert((head) != NULL); \ - assert((el) != NULL); \ - assert((add) != NULL); \ - if ((head) == (el)) { \ - (head) = (add); \ - (add)->next = (el)->next; \ - if ((el)->next == NULL) { \ - (add)->prev = (add); \ - } else { \ - (add)->prev = (el)->prev; \ - (add)->next->prev = (add); \ - } \ - } else { \ - (add)->next = (el)->next; \ - (add)->prev = (el)->prev; \ - (add)->prev->next = (add); \ - if ((el)->next == NULL) { \ - (head)->prev = (add); \ - } else { \ - (add)->next->prev = (add); \ - } \ - } \ -} while (0) - -#define DL_REPLACE_ELEM(head, el, add) \ - DL_REPLACE_ELEM2(head, el, add, prev, next) - -#define DL_PREPEND_ELEM2(head, el, add, prev, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el); \ - (add)->prev = (el)->prev; \ - (el)->prev = (add); \ - if ((head) == (el)) { \ - (head) = (add); \ - } else { \ - (add)->prev->next = (add); \ - } \ - } else { \ - DL_APPEND2(head, add, prev, next); \ - } \ -} while (0) \ - -#define DL_PREPEND_ELEM(head, el, add) \ - DL_PREPEND_ELEM2(head, el, add, prev, next) - -#define DL_APPEND_ELEM2(head, el, add, prev, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el)->next; \ - (add)->prev = (el); \ - (el)->next = (add); \ - if ((add)->next) { \ - (add)->next->prev = (add); \ - } else { \ - (head)->prev = (add); \ - } \ - } else { \ - DL_PREPEND2(head, add, prev, next); \ - } \ -} while (0) \ - -#define DL_APPEND_ELEM(head, el, add) \ - DL_APPEND_ELEM2(head, el, add, prev, next) - -#ifdef NO_DECLTYPE -/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ - -#undef DL_INSERT_INORDER2 -#define DL_INSERT_INORDER2(head,add,cmp,prev,next) \ -do { \ - if ((head) == NULL) { \ - (add)->prev = (add); \ - (add)->next = NULL; \ - (head) = (add); \ - } else if ((cmp(head, add)) >= 0) { \ - (add)->prev = (head)->prev; \ - (add)->next = (head); \ - (head)->prev = (add); \ - (head) = (add); \ - } else { \ - char *_tmp = (char*)(head); \ - while ((head)->next && (cmp((head)->next, add)) < 0) { \ - (head) = (head)->next; \ - } \ - (add)->prev = (head); \ - (add)->next = (head)->next; \ - (head)->next = (add); \ - UTLIST_RS(head); \ - if ((add)->next) { \ - (add)->next->prev = (add); \ - } else { \ - (head)->prev = (add); \ - } \ - } \ -} while (0) -#endif /* NO_DECLTYPE */ - -/****************************************************************************** - * circular doubly linked list macros * - *****************************************************************************/ -#define CDL_APPEND(head,add) \ - CDL_APPEND2(head,add,prev,next) - -#define CDL_APPEND2(head,add,prev,next) \ -do { \ - if (head) { \ - (add)->prev = (head)->prev; \ - (add)->next = (head); \ - (head)->prev = (add); \ - (add)->prev->next = (add); \ - } else { \ - (add)->prev = (add); \ - (add)->next = (add); \ - (head) = (add); \ - } \ -} while (0) - -#define CDL_PREPEND(head,add) \ - CDL_PREPEND2(head,add,prev,next) - -#define CDL_PREPEND2(head,add,prev,next) \ -do { \ - if (head) { \ - (add)->prev = (head)->prev; \ - (add)->next = (head); \ - (head)->prev = (add); \ - (add)->prev->next = (add); \ - } else { \ - (add)->prev = (add); \ - (add)->next = (add); \ - } \ - (head) = (add); \ -} while (0) - -#define CDL_INSERT_INORDER(head,add,cmp) \ - CDL_INSERT_INORDER2(head,add,cmp,prev,next) - -#define CDL_INSERT_INORDER2(head,add,cmp,prev,next) \ -do { \ - LDECLTYPE(head) _tmp; \ - if (head) { \ - CDL_LOWER_BOUND2(head, _tmp, add, cmp, next); \ - CDL_APPEND_ELEM2(head, _tmp, add, prev, next); \ - } else { \ - (head) = (add); \ - (head)->next = (head); \ - (head)->prev = (head); \ - } \ -} while (0) - -#define CDL_LOWER_BOUND(head,elt,like,cmp) \ - CDL_LOWER_BOUND2(head,elt,like,cmp,next) - -#define CDL_LOWER_BOUND2(head,elt,like,cmp,next) \ -do { \ - if ((head) == NULL || (cmp(head, like)) >= 0) { \ - (elt) = NULL; \ - } else { \ - for ((elt) = (head); (elt)->next != (head); (elt) = (elt)->next) { \ - if ((cmp((elt)->next, like)) >= 0) { \ - break; \ - } \ - } \ - } \ -} while (0) - -#define CDL_DELETE(head,del) \ - CDL_DELETE2(head,del,prev,next) - -#define CDL_DELETE2(head,del,prev,next) \ -do { \ - if (((head)==(del)) && ((head)->next == (head))) { \ - (head) = NULL; \ - } else { \ - (del)->next->prev = (del)->prev; \ - (del)->prev->next = (del)->next; \ - if ((del) == (head)) (head)=(del)->next; \ - } \ -} while (0) - -#define CDL_COUNT(head,el,counter) \ - CDL_COUNT2(head,el,counter,next) \ - -#define CDL_COUNT2(head, el, counter,next) \ -do { \ - (counter) = 0; \ - CDL_FOREACH2(head,el,next) { ++(counter); } \ -} while (0) - -#define CDL_FOREACH(head,el) \ - CDL_FOREACH2(head,el,next) - -#define CDL_FOREACH2(head,el,next) \ - for ((el)=(head);el;(el)=(((el)->next==(head)) ? NULL : (el)->next)) - -#define CDL_FOREACH_SAFE(head,el,tmp1,tmp2) \ - CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) - -#define CDL_FOREACH_SAFE2(head,el,tmp1,tmp2,prev,next) \ - for ((el) = (head), (tmp1) = (head) ? (head)->prev : NULL; \ - (el) && ((tmp2) = (el)->next, 1); \ - (el) = ((el) == (tmp1) ? NULL : (tmp2))) - -#define CDL_SEARCH_SCALAR(head,out,field,val) \ - CDL_SEARCH_SCALAR2(head,out,field,val,next) - -#define CDL_SEARCH_SCALAR2(head,out,field,val,next) \ -do { \ - CDL_FOREACH2(head,out,next) { \ - if ((out)->field == (val)) break; \ - } \ -} while (0) - -#define CDL_SEARCH(head,out,elt,cmp) \ - CDL_SEARCH2(head,out,elt,cmp,next) - -#define CDL_SEARCH2(head,out,elt,cmp,next) \ -do { \ - CDL_FOREACH2(head,out,next) { \ - if ((cmp(out,elt))==0) break; \ - } \ -} while (0) - -#define CDL_REPLACE_ELEM2(head, el, add, prev, next) \ -do { \ - assert((head) != NULL); \ - assert((el) != NULL); \ - assert((add) != NULL); \ - if ((el)->next == (el)) { \ - (add)->next = (add); \ - (add)->prev = (add); \ - (head) = (add); \ - } else { \ - (add)->next = (el)->next; \ - (add)->prev = (el)->prev; \ - (add)->next->prev = (add); \ - (add)->prev->next = (add); \ - if ((head) == (el)) { \ - (head) = (add); \ - } \ - } \ -} while (0) - -#define CDL_REPLACE_ELEM(head, el, add) \ - CDL_REPLACE_ELEM2(head, el, add, prev, next) - -#define CDL_PREPEND_ELEM2(head, el, add, prev, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el); \ - (add)->prev = (el)->prev; \ - (el)->prev = (add); \ - (add)->prev->next = (add); \ - if ((head) == (el)) { \ - (head) = (add); \ - } \ - } else { \ - CDL_APPEND2(head, add, prev, next); \ - } \ -} while (0) - -#define CDL_PREPEND_ELEM(head, el, add) \ - CDL_PREPEND_ELEM2(head, el, add, prev, next) - -#define CDL_APPEND_ELEM2(head, el, add, prev, next) \ -do { \ - if (el) { \ - assert((head) != NULL); \ - assert((add) != NULL); \ - (add)->next = (el)->next; \ - (add)->prev = (el); \ - (el)->next = (add); \ - (add)->next->prev = (add); \ - } else { \ - CDL_PREPEND2(head, add, prev, next); \ - } \ -} while (0) - -#define CDL_APPEND_ELEM(head, el, add) \ - CDL_APPEND_ELEM2(head, el, add, prev, next) - -#ifdef NO_DECLTYPE -/* Here are VS2008 / NO_DECLTYPE replacements for a few functions */ - -#undef CDL_INSERT_INORDER2 -#define CDL_INSERT_INORDER2(head,add,cmp,prev,next) \ -do { \ - if ((head) == NULL) { \ - (add)->prev = (add); \ - (add)->next = (add); \ - (head) = (add); \ - } else if ((cmp(head, add)) >= 0) { \ - (add)->prev = (head)->prev; \ - (add)->next = (head); \ - (add)->prev->next = (add); \ - (head)->prev = (add); \ - (head) = (add); \ - } else { \ - char *_tmp = (char*)(head); \ - while ((char*)(head)->next != _tmp && (cmp((head)->next, add)) < 0) { \ - (head) = (head)->next; \ - } \ - (add)->prev = (head); \ - (add)->next = (head)->next; \ - (add)->next->prev = (add); \ - (head)->next = (add); \ - UTLIST_RS(head); \ - } \ -} while (0) -#endif /* NO_DECLTYPE */ - -#endif /* UTLIST_H */ diff --git a/src/inc_internal/uthash/utringbuffer.h b/src/inc_internal/uthash/utringbuffer.h deleted file mode 100644 index ce2890e..0000000 --- a/src/inc_internal/uthash/utringbuffer.h +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright (c) 2015-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* a ring-buffer implementation using macros - */ -#ifndef UTRINGBUFFER_H -#define UTRINGBUFFER_H - -#define UTRINGBUFFER_VERSION 2.1.0 - -#include -#include -#include "utarray.h" // for "UT_icd" - -typedef struct { - unsigned i; /* index of next available slot; wraps at n */ - unsigned n; /* capacity */ - unsigned char f; /* full */ - UT_icd icd; /* initializer, copy and destructor functions */ - char *d; /* n slots of size icd->sz */ -} UT_ringbuffer; - -#define utringbuffer_init(a, _n, _icd) do { \ - memset(a, 0, sizeof(UT_ringbuffer)); \ - (a)->icd = *(_icd); \ - (a)->n = (_n); \ - if ((a)->n) { (a)->d = (char*)malloc((a)->n * (_icd)->sz); } \ -} while(0) - -#define utringbuffer_clear(a) do { \ - if ((a)->icd.dtor) { \ - if ((a)->f) { \ - unsigned _ut_i; \ - for (_ut_i = 0; _ut_i < (a)->n; ++_ut_i) { \ - (a)->icd.dtor(utringbuffer_eltptr(a, _ut_i)); \ - } \ - } else { \ - unsigned _ut_i; \ - for (_ut_i = 0; _ut_i < (a)->i; ++_ut_i) { \ - (a)->icd.dtor(utringbuffer_eltptr(a, _ut_i)); \ - } \ - } \ - } \ - (a)->i = 0; \ - (a)->f = 0; \ -} while(0) - -#define utringbuffer_done(a) do { \ - utringbuffer_clear(a); \ - free((a)->d); (a)->d = NULL; \ - (a)->n = 0; \ -} while(0) - -#define utringbuffer_new(a,n,_icd) do { \ - a = (UT_ringbuffer*)malloc(sizeof(UT_ringbuffer)); \ - utringbuffer_init(a, n, _icd); \ -} while(0) - -#define utringbuffer_free(a) do { \ - utringbuffer_done(a); \ - free(a); \ -} while(0) - -#define utringbuffer_push_back(a,p) do { \ - if ((a)->icd.dtor && (a)->f) { (a)->icd.dtor(_utringbuffer_internalptr(a,(a)->i)); } \ - if ((a)->icd.copy) { (a)->icd.copy( _utringbuffer_internalptr(a,(a)->i), p); } \ - else { memcpy(_utringbuffer_internalptr(a,(a)->i), p, (a)->icd.sz); }; \ - if (++(a)->i == (a)->n) { (a)->i = 0; (a)->f = 1; } \ -} while(0) - -#define utringbuffer_len(a) ((a)->f ? (a)->n : (a)->i) -#define utringbuffer_empty(a) ((a)->i == 0 && !(a)->f) -#define utringbuffer_full(a) ((a)->f != 0) - -#define _utringbuffer_real_idx(a,j) ((a)->f ? ((j) + (a)->i) % (a)->n : (j)) -#define _utringbuffer_internalptr(a,j) ((void*)((a)->d + ((a)->icd.sz * (j)))) -#define utringbuffer_eltptr(a,j) ((0 <= (j) && (j) < utringbuffer_len(a)) ? _utringbuffer_internalptr(a,_utringbuffer_real_idx(a,j)) : NULL) - -#define _utringbuffer_fake_idx(a,j) ((a)->f ? ((j) + (a)->n - (a)->i) % (a)->n : (j)) -#define _utringbuffer_internalidx(a,e) (((char*)(e) >= (a)->d) ? (((char*)(e) - (a)->d)/(a)->icd.sz) : -1) -#define utringbuffer_eltidx(a,e) _utringbuffer_fake_idx(a, _utringbuffer_internalidx(a,e)) - -#define utringbuffer_front(a) utringbuffer_eltptr(a,0) -#define utringbuffer_next(a,e) ((e)==NULL ? utringbuffer_front(a) : utringbuffer_eltptr(a, utringbuffer_eltidx(a,e)+1)) -#define utringbuffer_prev(a,e) ((e)==NULL ? utringbuffer_back(a) : utringbuffer_eltptr(a, utringbuffer_eltidx(a,e)-1)) -#define utringbuffer_back(a) (utringbuffer_empty(a) ? NULL : utringbuffer_eltptr(a, utringbuffer_len(a) - 1)) - -#endif /* UTRINGBUFFER_H */ diff --git a/src/inc_internal/uthash/utstack.h b/src/inc_internal/uthash/utstack.h deleted file mode 100644 index 3b0c1a0..0000000 --- a/src/inc_internal/uthash/utstack.h +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright (c) 2018-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef UTSTACK_H -#define UTSTACK_H - -#define UTSTACK_VERSION 2.1.0 - -/* - * This file contains macros to manipulate a singly-linked list as a stack. - * - * To use utstack, your structure must have a "next" pointer. - * - * ----------------.EXAMPLE ------------------------- - * struct item { - * int id; - * struct item *next; - * } - * - * struct item *stack = NULL: - * - * int main() { - * int count; - * struct item *tmp; - * struct item *item = malloc(sizeof *item); - * item->id = 42; - * STACK_COUNT(stack, tmp, count); assert(count == 0); - * STACK_PUSH(stack, item); - * STACK_COUNT(stack, tmp, count); assert(count == 1); - * STACK_POP(stack, item); - * free(item); - * STACK_COUNT(stack, tmp, count); assert(count == 0); - * } - * -------------------------------------------------- - */ - -#define STACK_TOP(head) (head) - -#define STACK_EMPTY(head) (!(head)) - -#define STACK_PUSH(head,add) \ - STACK_PUSH2(head,add,next) - -#define STACK_PUSH2(head,add,next) \ -do { \ - (add)->next = (head); \ - (head) = (add); \ -} while (0) - -#define STACK_POP(head,result) \ - STACK_POP2(head,result,next) - -#define STACK_POP2(head,result,next) \ -do { \ - (result) = (head); \ - (head) = (head)->next; \ -} while (0) - -#define STACK_COUNT(head,el,counter) \ - STACK_COUNT2(head,el,counter,next) \ - -#define STACK_COUNT2(head,el,counter,next) \ -do { \ - (counter) = 0; \ - for ((el) = (head); el; (el) = (el)->next) { ++(counter); } \ -} while (0) - -#endif /* UTSTACK_H */ diff --git a/src/inc_internal/uthash/utstring.h b/src/inc_internal/uthash/utstring.h deleted file mode 100644 index 4cf5ffd..0000000 --- a/src/inc_internal/uthash/utstring.h +++ /dev/null @@ -1,407 +0,0 @@ -/* -Copyright (c) 2008-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -/* a dynamic string implementation using macros - */ -#ifndef UTSTRING_H -#define UTSTRING_H - -#define UTSTRING_VERSION 2.1.0 - -#include -#include -#include -#include - -#ifdef __GNUC__ -#define UTSTRING_UNUSED __attribute__((__unused__)) -#else -#define UTSTRING_UNUSED -#endif - -#ifdef oom -#error "The name of macro 'oom' has been changed to 'utstring_oom'. Please update your code." -#define utstring_oom() oom() -#endif - -#ifndef utstring_oom -#define utstring_oom() exit(-1) -#endif - -typedef struct { - char *d; /* pointer to allocated buffer */ - size_t n; /* allocated capacity */ - size_t i; /* index of first unused byte */ -} UT_string; - -#define utstring_reserve(s,amt) \ -do { \ - if (((s)->n - (s)->i) < (size_t)(amt)) { \ - char *utstring_tmp = (char*)realloc( \ - (s)->d, (s)->n + (amt)); \ - if (!utstring_tmp) { \ - utstring_oom(); \ - } \ - (s)->d = utstring_tmp; \ - (s)->n += (amt); \ - } \ -} while(0) - -#define utstring_init(s) \ -do { \ - (s)->n = 0; (s)->i = 0; (s)->d = NULL; \ - utstring_reserve(s,100); \ - (s)->d[0] = '\0'; \ -} while(0) - -#define utstring_done(s) \ -do { \ - if ((s)->d != NULL) free((s)->d); \ - (s)->n = 0; \ -} while(0) - -#define utstring_free(s) \ -do { \ - utstring_done(s); \ - free(s); \ -} while(0) - -#define utstring_new(s) \ -do { \ - (s) = (UT_string*)malloc(sizeof(UT_string)); \ - if (!(s)) { \ - utstring_oom(); \ - } \ - utstring_init(s); \ -} while(0) - -#define utstring_renew(s) \ -do { \ - if (s) { \ - utstring_clear(s); \ - } else { \ - utstring_new(s); \ - } \ -} while(0) - -#define utstring_clear(s) \ -do { \ - (s)->i = 0; \ - (s)->d[0] = '\0'; \ -} while(0) - -#define utstring_bincpy(s,b,l) \ -do { \ - utstring_reserve((s),(l)+1); \ - if (l) memcpy(&(s)->d[(s)->i], b, l); \ - (s)->i += (l); \ - (s)->d[(s)->i]='\0'; \ -} while(0) - -#define utstring_concat(dst,src) \ -do { \ - utstring_reserve((dst),((src)->i)+1); \ - if ((src)->i) memcpy(&(dst)->d[(dst)->i], (src)->d, (src)->i); \ - (dst)->i += (src)->i; \ - (dst)->d[(dst)->i]='\0'; \ -} while(0) - -#define utstring_len(s) ((s)->i) - -#define utstring_body(s) ((s)->d) - -UTSTRING_UNUSED static void utstring_printf_va(UT_string *s, const char *fmt, va_list ap) { - int n; - va_list cp; - for (;;) { -#ifdef _WIN32 - cp = ap; -#else - va_copy(cp, ap); -#endif - n = vsnprintf (&s->d[s->i], s->n-s->i, fmt, cp); - va_end(cp); - - if ((n > -1) && ((size_t) n < (s->n-s->i))) { - s->i += n; - return; - } - - /* Else try again with more space. */ - if (n > -1) utstring_reserve(s,n+1); /* exact */ - else utstring_reserve(s,(s->n)*2); /* 2x */ - } -} -#ifdef __GNUC__ -/* support printf format checking (2=the format string, 3=start of varargs) */ -static void utstring_printf(UT_string *s, const char *fmt, ...) - __attribute__ (( format( printf, 2, 3) )); -#endif -UTSTRING_UNUSED static void utstring_printf(UT_string *s, const char *fmt, ...) { - va_list ap; - va_start(ap,fmt); - utstring_printf_va(s,fmt,ap); - va_end(ap); -} - -/******************************************************************************* - * begin substring search functions * - ******************************************************************************/ -/* Build KMP table from left to right. */ -UTSTRING_UNUSED static void _utstring_BuildTable( - const char *P_Needle, - size_t P_NeedleLen, - long *P_KMP_Table) -{ - long i, j; - - i = 0; - j = i - 1; - P_KMP_Table[i] = j; - while (i < (long) P_NeedleLen) - { - while ( (j > -1) && (P_Needle[i] != P_Needle[j]) ) - { - j = P_KMP_Table[j]; - } - i++; - j++; - if (i < (long) P_NeedleLen) - { - if (P_Needle[i] == P_Needle[j]) - { - P_KMP_Table[i] = P_KMP_Table[j]; - } - else - { - P_KMP_Table[i] = j; - } - } - else - { - P_KMP_Table[i] = j; - } - } - - return; -} - - -/* Build KMP table from right to left. */ -UTSTRING_UNUSED static void _utstring_BuildTableR( - const char *P_Needle, - size_t P_NeedleLen, - long *P_KMP_Table) -{ - long i, j; - - i = P_NeedleLen - 1; - j = i + 1; - P_KMP_Table[i + 1] = j; - while (i >= 0) - { - while ( (j < (long) P_NeedleLen) && (P_Needle[i] != P_Needle[j]) ) - { - j = P_KMP_Table[j + 1]; - } - i--; - j--; - if (i >= 0) - { - if (P_Needle[i] == P_Needle[j]) - { - P_KMP_Table[i + 1] = P_KMP_Table[j + 1]; - } - else - { - P_KMP_Table[i + 1] = j; - } - } - else - { - P_KMP_Table[i + 1] = j; - } - } - - return; -} - - -/* Search data from left to right. ( Multiple search mode. ) */ -UTSTRING_UNUSED static long _utstring_find( - const char *P_Haystack, - size_t P_HaystackLen, - const char *P_Needle, - size_t P_NeedleLen, - long *P_KMP_Table) -{ - long i, j; - long V_FindPosition = -1; - - /* Search from left to right. */ - i = j = 0; - while ( (j < (int)P_HaystackLen) && (((P_HaystackLen - j) + i) >= P_NeedleLen) ) - { - while ( (i > -1) && (P_Needle[i] != P_Haystack[j]) ) - { - i = P_KMP_Table[i]; - } - i++; - j++; - if (i >= (int)P_NeedleLen) - { - /* Found. */ - V_FindPosition = j - i; - break; - } - } - - return V_FindPosition; -} - - -/* Search data from right to left. ( Multiple search mode. ) */ -UTSTRING_UNUSED static long _utstring_findR( - const char *P_Haystack, - size_t P_HaystackLen, - const char *P_Needle, - size_t P_NeedleLen, - long *P_KMP_Table) -{ - long i, j; - long V_FindPosition = -1; - - /* Search from right to left. */ - j = (P_HaystackLen - 1); - i = (P_NeedleLen - 1); - while ( (j >= 0) && (j >= i) ) - { - while ( (i < (int)P_NeedleLen) && (P_Needle[i] != P_Haystack[j]) ) - { - i = P_KMP_Table[i + 1]; - } - i--; - j--; - if (i < 0) - { - /* Found. */ - V_FindPosition = j + 1; - break; - } - } - - return V_FindPosition; -} - - -/* Search data from left to right. ( One time search mode. ) */ -UTSTRING_UNUSED static long utstring_find( - UT_string *s, - long P_StartPosition, /* Start from 0. -1 means last position. */ - const char *P_Needle, - size_t P_NeedleLen) -{ - long V_StartPosition; - long V_HaystackLen; - long *V_KMP_Table; - long V_FindPosition = -1; - - if (P_StartPosition < 0) - { - V_StartPosition = s->i + P_StartPosition; - } - else - { - V_StartPosition = P_StartPosition; - } - V_HaystackLen = s->i - V_StartPosition; - if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) - { - V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); - if (V_KMP_Table != NULL) - { - _utstring_BuildTable(P_Needle, P_NeedleLen, V_KMP_Table); - - V_FindPosition = _utstring_find(s->d + V_StartPosition, - V_HaystackLen, - P_Needle, - P_NeedleLen, - V_KMP_Table); - if (V_FindPosition >= 0) - { - V_FindPosition += V_StartPosition; - } - - free(V_KMP_Table); - } - } - - return V_FindPosition; -} - - -/* Search data from right to left. ( One time search mode. ) */ -UTSTRING_UNUSED static long utstring_findR( - UT_string *s, - long P_StartPosition, /* Start from 0. -1 means last position. */ - const char *P_Needle, - size_t P_NeedleLen) -{ - long V_StartPosition; - long V_HaystackLen; - long *V_KMP_Table; - long V_FindPosition = -1; - - if (P_StartPosition < 0) - { - V_StartPosition = s->i + P_StartPosition; - } - else - { - V_StartPosition = P_StartPosition; - } - V_HaystackLen = V_StartPosition + 1; - if ( (V_HaystackLen >= (long) P_NeedleLen) && (P_NeedleLen > 0) ) - { - V_KMP_Table = (long *)malloc(sizeof(long) * (P_NeedleLen + 1)); - if (V_KMP_Table != NULL) - { - _utstring_BuildTableR(P_Needle, P_NeedleLen, V_KMP_Table); - - V_FindPosition = _utstring_findR(s->d, - V_HaystackLen, - P_Needle, - P_NeedleLen, - V_KMP_Table); - - free(V_KMP_Table); - } - } - - return V_FindPosition; -} -/******************************************************************************* - * end substring search functions * - ******************************************************************************/ - -#endif /* UTSTRING_H */ diff --git a/src/inc_internal/view_only/IPMatcher.h b/src/inc_internal/view_only/IPMatcher.h deleted file mode 100644 index 0e387f3..0000000 --- a/src/inc_internal/view_only/IPMatcher.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * - * Copyright (c) 2020 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LU YUHAI (luyuhai@iie.ac.cn) - * Last modification: 2020-04-20 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef H_IP_MATCHER_H -#define H_IP_MATCHER_H -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - - enum IP_TYPE - { - IPv4, - IPv6 - }; - - /* ´øÑÚÂëµÄµ¥µãIPv4¹æÔò */ - struct ipv4_range - { - unsigned int start_ip; /* IP·¶Î§Ï½ç */ - unsigned int end_ip; /* IP·¶Î§ÉϽç */ - }; - - /* ´øÑÚÂëµÄµ¥µãIPv6¹æÔò */ - struct ipv6_range - { - unsigned int start_ip[4]; /* IP·¶Î§Ï½ç */ - unsigned int end_ip[4]; /* IP·¶Î§ÉϽç */ - }; - - /* ͨÓõÄip¹æÔòÀàÐÍ */ - struct ip_rule - { - enum IP_TYPE type; /* ¹æÔòÀàÐÍ£¬ipv4»òipv6 */ - unsigned int rule_id; /* ¹æÔòID */ - void* user_tag; /* Óû§×Ô¶¨ÒåÊý¾Ý£¬ÃüÖÐʱËæÆ¥Åä½á¹û·µ»Ø */ - union - { - struct ipv4_range ipv4_rule; /*´øÑÚÂëµÄµ¥µãIPv4¹æÔò*/ - struct ipv6_range ipv6_rule; /*´øÑÚÂëµÄµ¥µãIPv6¹æÔò*/ - }; - }; - - /* ͨÓõĴýɨÃèÊý¾ÝÀàÐÍ */ - struct ip_data - { - enum IP_TYPE type; /* ¹æÔòÀàÐÍ£¬ipv4»òipv6 */ - union /* ¸ù¾Ýrule_type¾ö¶¨Êý¾Ý¸ºÔØÊÇipv4»¹ÊÇipv6 */ - { - unsigned int ipv4; /* ipv4Êý¾Ý*/ - unsigned int ipv6[4]; /* ipv6Êý¾Ý*/ - }; - }; - - - /* ²¼¶û±í´ïʽµÄɨÃè½á¹ûÀàÐÍ */ - struct scan_result - { - unsigned int rule_id; /* ¹æÔòµÄID */ - void * tag; /* Óû§×Ô¶¨ÒåÊý¾Ý£¬ÃüÖÐʱËæÆ¥Åä½á¹û·µ»Ø */ - }; - - - struct ip_matcher; - - /* - ¹¦ÄÜ£º¸ù¾ÝÊäÈëµÄ¹æÔòÉú³ÉɨÃèÆ÷ - ²ÎÊý£º - rules[in]£ºÒ»×éip¹æÔò - rule_num[in]£ºÊäÈëµÄ¹æÔòÊýÁ¿ - mem_use[out]£ºÄÚ´æÏûºÄÁ¿ - ·µ»ØÖµ£º - ipɨÃèÆ÷,·µ»Ø¿ÕÖ¸ÕëÉú³ÉɨÃèÆ÷ʧ°Ü - */ - struct ip_matcher* ip_matcher_new(struct ip_rule * rules, size_t rule_num, size_t * mem_use); - - /* - ¹¦ÄÜ£ºµ÷ÓÃipɨÃèÆ÷¶ÔÊäÈëµÄipÊý¾Ý½øÐÐɨÃè - ²ÎÊý£º - matcher[in]£ºipɨÃèÆ÷ - data[in]£ºÊäÈëµÄ´ýɨÃèipÊý¾Ý - result[in]£º·µ»Ø½á¹û´æ´¢Êý×é - size[in]£º½á¹ûÊý×éµÄ´óС - ·µ»ØÖµ£º - ÃüÖнá¹ûµÄÊýÁ¿£¨<=size£©£»·µ»ØֵΪ-1±íʾ³ö´í¡£ - - */ - int ip_matcher_match(struct ip_matcher* matcher, struct ip_data * data, struct scan_result* result, size_t size); - - /* - ¹¦ÄÜ£ºÏú»ÙÒ»¸öipɨÃèÆ÷ - ²ÎÊý£º - matcher[in]£º´ýÏú»ÙµÄipɨÃèÆ÷Ö¸Õë - */ - void ip_matcher_free(struct ip_matcher* matcher); - -#ifdef __cplusplus -} -#endif - -#endif /* !defined(H_IP_MATCHER_H) */ diff --git a/src/inc_internal/view_only/MESA_handle_logger.h b/src/inc_internal/view_only/MESA_handle_logger.h deleted file mode 100644 index c7e031d..0000000 --- a/src/inc_internal/view_only/MESA_handle_logger.h +++ /dev/null @@ -1,75 +0,0 @@ -#ifndef MESA_HANDLE__LOGGER_H -#define MESA_HANDLE__LOGGER_H - -/* - * runtime_log with handle, - * based on runtime_log. - * yang wei - * create time:2014-03-24 - * version:20140324 - */ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include -#include -#include -#include -#include -#include - -#define RLOG_LV_DEBUG 10 -#define RLOG_LV_INFO 20 -#define RLOG_LV_FATAL 30 - - -#define MESA_HANDLE_RUNTIME_LOG(handle, lv, mod, fmt, args...) \ - MESA_handle_runtime_log((handle), (lv), (mod), "file %s, line %d, " fmt, \ - __FILE__, __LINE__, ##args) - -/* - * name: MESA_create_runtime_log_handle - * functionality: get runtime_log handle; - * params: - * file_path: path of log file; - * level: level of log; - * returns: - * not NULL, if succeeded; - * NULL, if file is not absolute path, or failed to create log file; - */ -void *MESA_create_runtime_log_handle(const char *file_path, int level); - -/* - * name: MESA_handle_runtime_log - * functionality: appends log message to runtime log file; - * params: - * handle:handle of runtime log, which is created by MESA_create_runtime_log_handle; - * level: log level, messages with level value smaller the global var - * "runtime_log_level" are ignored; - * module: name of loggin module; - * fmt: format string; - * returns: - * none; - */ -void MESA_handle_runtime_log(void *handle, int level, const char *module, const char *fmt, ...); - -/* - * name: MESA_destroy_runtime_log_handle - * functionality: release runtime log handle memory. - * params: - * handle: runtime log handle which is going to be released; - * returns: - * none; - */ -void MESA_destroy_runtime_log_handle(void *handle); - -#ifdef __cplusplus -} -#endif - -#endif - - diff --git a/src/inc_internal/view_only/MESA_htable.h b/src/inc_internal/view_only/MESA_htable.h deleted file mode 100644 index ed5a074..0000000 --- a/src/inc_internal/view_only/MESA_htable.h +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef _MESA_HASH_V3_H_ -#define _MESA_HASH_V3_H_ -#ifdef __cplusplus -extern "C" -{ -#endif - -#include -#include -#include -#include - -/* - * general purpose hash table implementation. - * - * xiang hong - * 2002-07-28 - *History: - * 2012-03-23 zhengchao add thread safe option and link expire feature; - * 2014-01-27 lijia add reentrant feature. - */ - -#define COMPLEX_KEY_SWITCH (1) - -#define MESA_HASH_DEBUG (0) - -#define ELIMINATE_TYPE_NUM (1) -#define ELIMINATE_TYPE_TIME (2) -#define ELIMINATE_TYPE_MANUAL (3) /* delete oldest item by manual */ - -typedef void * MESA_htable_handle; - - -#define HASH_MALLOC(_n_) malloc(_n_) -#define HASH_FREE(_p_) free(_p_) - - -#if 1 -#define HASH_TIME_NOW() time(NULL) -#else -extern volatile time_t g_CurrentTime; /* ´Ë±äÁ¿ÔÚÁíÒ»¸öÏß³ÌÖÐÿ¸ô1Ãë×ÔÔöÒ»´Î */ -#define HASH_TIME_NOW() (time_t)g_CurrentTime -#endif - -#ifndef uchar -#define uchar unsigned char -#endif -#ifndef uint -#define uint unsigned int -#endif - -/* eliminate algorithm */ -#define HASH_ELIMINATE_ALGO_FIFO (0) /* by default */ -#define HASH_ELIMINATE_ALGO_LRU (1) - -/* - * hash key compare function prototype, see hash_key_comp(). - * return value: - * 0:key1 and key2 are equal; - * other:key1 and key2 not equal. - */ -typedef int key_comp_fun_t(const uchar * key1, uint size1, const uchar * key2, uint size2); - -/* - * hash key->index computing function prototype, see hash_key2index(). - */ -typedef uint key2index_fun_t(const MESA_htable_handle table, const uchar * key, uint size); - -typedef long hash_cb_fun_t(void *data, const uchar *key, uint size, void *user_arg); - -/* - * thread_safe: 0:create hash table without thread safe features; - * positive:the bigger number has more performance, less collide, but less timeout accuracy. - * max number is 1024. - * recursive: 0:can't recursive call MESA_htable_xxx series function - * 1:can recursive call MESA_htable_xxx series function. - * hash_slot_size: how big do you want the table to be, must be 2^N; - * max_elem_num: the maximum elements of the HASH-table,0 means infinite; - * key_comp: hash key compare function, use default function if NULL; - * suggest implement by yourself. - * key2index: hash key->index computing function, use default function if NULL; - * suggest use MESA_htable built-in function. - * data_free: release resources function, only free attached data pointer if NULL; - * data_expire_with_condition: if expire_time > 0, call this function when a element expire, eliminate always if NULL; - * args: - * data: pointer to attached data; - * type: eliminate reason, ELIMINATE_TYPE_NUM or ELIMINATE_TYPE_TIME; - * return value of 'data_expire_with_condition': - * 1: can be eliminated; - * 0: can't be eliminated, renew the item. - * eliminate_type: the algorithm of elimanate a expired element, 0:FIFO; 1:LRU. - * expire_time: the element expire time in second, 0 means infinite. - */ -typedef struct{ - unsigned int thread_safe; - int recursive; - unsigned int hash_slot_size; - unsigned int max_elem_num; - int eliminate_type; - int expire_time; - key_comp_fun_t * key_comp; - key2index_fun_t * key2index; - void (* data_free)(void *data); - int (*data_expire_with_condition)(void *data, int type); -#if COMPLEX_KEY_SWITCH - uchar* (*complex_key_dup)(const uchar *key, uint key_size); - void (* complex_key_free)(uchar *key, uint key_size); -#endif -}MESA_htable_create_args_t; - -/* - * name: MESA_htable_create - * functionality: allocats memory for hash slots, and initialize hash structure; - * param: - * args: argments set; - * args_len: length of argment set; - * returns: - * NULL : error; - * Non-NULL : success; - */ -MESA_htable_handle MESA_htable_create(const MESA_htable_create_args_t *args, int args_struct_len); - -/* - * get total number of HASH element. -*/ -unsigned int MESA_htable_get_elem_num(const MESA_htable_handle table); - -/* - * name: MESA_htable_destroy - * functionality: cleans up hash structure, frees memory occupied; - * param: - * table: who is the victim; - * func: callback function to clean up data attached to hash items; - * returns: - * always returns 0; - */ -int MESA_htable_destroy(MESA_htable_handle table, void (* func)(void *)); - -/* - * name: MESA_htable_add - * functionality: adds item to table, call hash_expire() if elem_count gets - * bigger than threshold_hi, and adjust threshold; - * param: - * table: to which table do you want to add; - * key: what is the label; - * size: how long is the label; - * data: what data do you want to attach; - * returns: - * >0 success,return hash elems' linklist size - * -1, duplicates found and can't add this one; - * -2, memory failure; - * -3, other errors. - */ -int MESA_htable_add(MESA_htable_handle table, const uchar * key, uint size, const void *data); -#if 0 -/* - * name: hash_add_with_expire - * functionality: adds item to table, than call hash_expire() on its list - * param: - * table: to which table do you want to add; - * key: what is the label; - * size: how long is the label; - * data: what data do you want to attach; - * returns: - * >0 success,return hash elems' linklist size - * -1, duplicates found and can't add this one; - * -2, memory failure; - */ -int MESA_hash_add_with_expire_v3(MESA_htable_inner_t * table, uchar * key, uint size, void * data); - -#endif - - -/* - * name: MESA_htable_del - * functionality: deletes item from table. - * param: - * table: from which table do you want to delete; - * key : what is the label; - * size : how long is the label; - * func : callback function to clean up data attached to hash items, - if this pointer is NULL will call "data_free" in MESA_hash_create(), - * returns: - * 0, success; - * -1, no such thing; - */ -int MESA_htable_del(MESA_htable_handle table, const uchar * key, uint size, - void (* func)(void *)); - -/* - * name: MESA_htable_del_oldest_manual - * functionality: deletes oldest item from table. - * param: - * table: from which table do you want to delete; - * func : callback function to clean up data attached to hash items, - if this pointer is NULL will call "data_free" in MESA_hash_create(), - * batch_num: delete oldest items. - * returns: - * 0, success; - * -1, no such thing; - */ -int MESA_htable_del_oldest_manual(MESA_htable_handle table, void (* func)(void *), int batch_num); - -/* - * name: MESA_htable_search - * functionality: selects item from table; - * param: - * table: from which table do you want to select; - * key : what is the label; - * size : how long is the label; - * - * return: - * not NULL :pointer to attached data; - * NULL :not found(thus be careful if you are attaching NULL data on purpose). - */ -void *MESA_htable_search(const MESA_htable_handle table, const uchar * key, uint size); - -/* - * name: MESA_htable_search_cb - * functionality: selects item from table, and then call 'cb', reentrant; - * in param: - * table: from which table do you want to select; - * key : what is the label; - * size : how long is the label; - * cb : call this function when found the attached data; - * arg : the argument of "cb" function. - * out param: - * cb_ret: the return value of the function "cb". - * return: - * not NULL :pointer to attached data; - * NULL :not found(thus be careful if you are attaching NULL data on purpose). - */ -void *MESA_htable_search_cb(const MESA_htable_handle table, const uchar * key, uint size, - hash_cb_fun_t *cb, void *arg, long *cb_ret); - -/* - * name: hash_iterate - * functionality: iterates each hash item; - * params: - * table: what table is to be iterated; - * func: what do you want to do to each attached data item; - * returns: - * 0: iterates all items; - * -1: error; - */ -int MESA_htable_iterate(MESA_htable_handle table, void (* func)(const uchar * key, uint size, void * data, void *user), void * user); - -/* - args: - print_switch: - 0: disable print message; - 1: enable print message; -*/ -void MESA_htable_print_crtl(MESA_htable_handle table, int print_switch); - -#if 0 -/* - * name: hash_expire - * functionality: iterates each item and deletes those that are expired; - * params: - * table: what table do you want to check; - * returns: - * always 0; - */ -int MESA_hash_expire(MESA_htable_inner_t * table); -#endif - - -#ifdef __cplusplus -} -#endif - -#endif /* _LIB_HASH_H_INCLUDED_ */ - - diff --git a/src/inc_internal/view_only/MESA_list.h b/src/inc_internal/view_only/MESA_list.h deleted file mode 100644 index 315e695..0000000 --- a/src/inc_internal/view_only/MESA_list.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef _LIST_COMMON_H_ -#define _LIST_COMMON_H_ - -typedef struct MESA_list{ - struct MESA_list *nextele; - struct MESA_list *preele; - void *quiddity; -}MESA_list_t; - -#ifdef __cplusplus -extern "C" -{ -#endif - -void MESA_list_init_head(struct MESA_list *head); -long MESA_list_get_count(const struct MESA_list *head); -int MESA_list_is_empty(const struct MESA_list *head); -void MESA_list_add(struct MESA_list *head, struct MESA_list *new_list); -void MESA_list_add_tail(struct MESA_list *head, struct MESA_list *new_list); -void MESA_list_del(struct MESA_list *head, struct MESA_list *del_list); -void MESA_list_move(struct MESA_list *head, struct MESA_list *list); -void MESA_list_move_tail(struct MESA_list *head, struct MESA_list *list); -struct MESA_list *MESA_list_join_n(struct MESA_list *head, struct MESA_list *op_place, struct MESA_list *new_obj); -struct MESA_list *MESA_list_join_p(struct MESA_list *head, struct MESA_list *new_obj, struct MESA_list *op_place); - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/src/inc_internal/view_only/MESA_list_queue.h b/src/inc_internal/view_only/MESA_list_queue.h deleted file mode 100644 index 88b830d..0000000 --- a/src/inc_internal/view_only/MESA_list_queue.h +++ /dev/null @@ -1,114 +0,0 @@ -#ifndef _MESA_LIST_V3_H_ -#define _MESA_LIST_V3_H_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#define MESA_LQUEUE_VERSION "MESA_lqueue_2014.03.24_12:00:00" - -/* - MESA_list µÚÈý°æ£¬ - 1-Ôö¼ÓḬ̈߳²È«ÌØÐÔ; - 2-Òþ²ØÄÚ²¿½á¹¹, ¸ü°²È«¡¢½Ó¿Ú¸ü¼ò½à; - 3-µ÷ÓÃÕßÎÞÐè×ÔÐйÜÀí½Úµã½á¹¹£¬Ê¹Óøü·½±ã; -*/ - - -#define MESA_LIST_OP_PLACE_HEAD (0x1) -#define MESA_LIST_OP_PLACE_TAIL (0x2) - -#define MESA_list_GET (0x1) -#define MESA_list_JOIN (0x2) - -#define MESA_list_BOLCK (0x4) -#define MESA_list_NONBOLCK (0x8) - -#define MESA_list_JOIN_BLOCK (MESA_list_JOIN|MESA_list_BOLCK) -#define MESA_list_JOIN_NONBLOCK (MESA_list_JOIN|MESA_list_NONBOLCK) -#define MESA_list_GET_BLOCK (MESA_list_GET|MESA_list_BOLCK) -#define MESA_list_GET_NONBLOCK (MESA_list_GET|MESA_list_NONBOLCK) - -typedef void * MESA_lqueue_head; -typedef int (* MESA_lqueue_cb_t)(void *data, long data_len, void *arg); - -/* All of the following functions return value */ -typedef enum{ - MESA_QUEUE_RET_OK = 0, /* success */ - MESA_QUEUE_RET_COMMON_ERR = -1, /* general¡¢undefined errors */ - MESA_QUEUE_RET_ARG_ERR = -2, /* invalid args */ - MESA_QUEUE_RET_NUM_FULL = -3, /* queue number full */ - MESA_QUEUE_RET_MEM_FULL = -4, /* queue memory full */ - MESA_QUEUE_RET_QEMPTY = -5, /* queue empty */ - MESA_QUEUE_RET_LEN_ERR = -6, /* length error */ - MESA_QUEUE_RET_CANT_GET_LOCK = -7, /* can't get lock in non-block mode */ - MESA_QUEUE_RET_GET_LOCK_TMOUT = -8, /* get lock timeout */ -}MESA_queue_errno_t; - -/* - args description: - [IN] - thread_safe : 1:create thread safe queue; 0:without thread safe insurance. - max_item_num: maximum queue items of the queue, 0 means infinity. -*/ -MESA_lqueue_head MESA_lqueue_create(int thread_safe, long max_item_num); - -/* - attention: - The follow two functions is get some value of queue in a moment, - however, the value you got is not exactly, - because it's maybe changed immediately by other thread when this functions is return. -*/ -long MESA_lqueue_get_mem_used(MESA_lqueue_head head); -long MESA_lqueue_get_count(MESA_lqueue_head head); - - -/* - args description: - [IN]: - lq_head : the handler of MESA_lqueue. - - [OUT]: - data : receive buffer. - - [IN && OUT]: - data_len: - is value-result argument, like "addrlen of recvfrom(2)", - the caller should initialize the size of the 'data', - will modified on return to indicate the actual size of the queue item. - -*/ -int MESA_lqueue_read_head(MESA_lqueue_head lq_head, void *data, long *data_len); -int MESA_lqueue_get_head(MESA_lqueue_head lqhead, void *data, long *data_len); - -/* - if return value of "cb" is 0, the behaviour is like MESA_lqueue_read_head(), - else if return value of "cb" is not 0, the behaviour is like MESA_lqueue_get_head(). -*/ -int MESA_lqueue_detect_get_head(MESA_lqueue_head lq_head, MESA_lqueue_cb_t cb, void *data, long *data_len, void *cb_arg); -int MESA_lqueue_get_tail(MESA_lqueue_head lq_head, void *data, long *data_len); - -int MESA_lqueue_join_head(MESA_lqueue_head lq_head, const void *data, long data_len); -int MESA_lqueue_join_tail(MESA_lqueue_head lq_head, const void *data, long data_len); - - -/* these functions features same with above no "try", - except shall return immediately, in other word is "Non-block mode"! - */ -int MESA_lqueue_try_read_head(MESA_lqueue_head lq_head, void *data, long *data_len); -int MESA_lqueue_try_get_head(MESA_lqueue_head lq_head, void *data, long *data_len); -int MESA_lqueue_try_get_tail(MESA_lqueue_head lq_head, void *data, long *data_len); -int MESA_lqueue_try_join_head(MESA_lqueue_head lq_head, const void *data, long data_len); -int MESA_lqueue_try_join_tail(MESA_lqueue_head lq_head, const void *data, long data_len); - - -void MESA_lqueue_destroy(MESA_lqueue_head head, MESA_lqueue_cb_t cb, void *cb_arg); - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/src/inc_internal/view_only/field_stat2.h b/src/inc_internal/view_only/field_stat2.h deleted file mode 100644 index f8bdc35..0000000 --- a/src/inc_internal/view_only/field_stat2.h +++ /dev/null @@ -1,119 +0,0 @@ -#ifndef H_SCREEN_STAT_H_INCLUDE -#define H_SCREEN_STAT_H_INCLUDE -#include -#include -#ifndef __cplusplus -#error("This file should be compiled with C++ compiler") -#endif - -enum field_dsp_style_t -{ - FS_STYLE_FIELD=0, - FS_STYLE_COLUMN, - FS_STYLE_LINE, - FS_STYLE_STATUS, - FS_STYLE_HISTOGRAM -}; -enum field_calc_algo -{ - FS_CALC_CURRENT=0, - FS_CALC_SPEED -}; -enum field_op -{ - FS_OP_ADD=1, - FS_OP_SET, - FS_OP_SUB -}; - -enum stats_output_format -{ - FS_OUTPUT_STATSD=1, - FS_OUTPUT_INFLUX_LINE=2 -}; - -enum metris_output_format -{ - FS_METRIS_OUTPUT_DEFAULT=0, - FS_METRIS_OUTPUT_JSON=1 -}; - -typedef void* screen_stat_handle_t; - -enum FS_option -{ - OUTPUT_DEVICE, //VALUE is a const char*, indicate a file path string, SIZE = strlen(string+'\0')+1.DEFAULT:output to stdout. - PRINT_MODE, //VALUE is an interger,1:Rewrite ,2: Append. SIZE=4,DEFALUT:REWRITE. - STAT_CYCLE, //VALUE is an interger idicate interval seconds of every output, SIZE=4 ,DEFUALT:2 seconds. - PRINT_TRIGGER, //VALUE is an interger,1:Do print,0: Don't print.SIZE=4.DEFAULT:1. - CREATE_THREAD, //VALUE is an interger,1: Create a print thread,0:not create,output by call passive_output function, - //and the STAT_CYCLE is meaningless.SIZE=4,DEFAULT:0. - ID_INVISBLE, //value is field_id/status_id/column_id, not output this string, SIZE=4,DEFAULT: shutdown NO one. - FLUSH_BY_DATE, //value is 1(true) or 0(false),SIZE=4,DEFAULT: Do not flush by date. - APP_NAME, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT is "?". - STATS_SERVER_IP, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT. - STATS_SERVER_PORT, //VALUE is a unsigned short or a signed int, host order, SIZE= sizeof(unsigned short) or sizeof(int). No DEFAULT. - STATS_FORMAT, //VALUE is enum stats_output_format, STATSD or INFLUX_LINE, SIZE=sizeof(int), DEFAULT:STATSD. - MAX_STAT_FIELD_NUM, //VALUE is an interger, SIZE=sizeof(int), DEFAULT:1024. - HISTOGRAM_GLOBAL_BINS, //VALUE is a const char*, define a histogram bins for default field, SIZE = strlen(string+'\0')+1. DEFAULT: "0.5,0.8,0.9,0.95,0.99“. - NOT_SEND_METRIC_TO_SERVER, //value is field_id/status_id/column_id, not output this string, SIZE=4,DEFAULT:Send. - METRIS_FORMAT, //VALUE is enum metris_output_format, DEFAULT or JSON, SIZE=sizeof(int). - OUTPUT_PROMETHEUS //VALUE is an interger,1:output prometheus ,0: not output. SIZE=4,DEFALUT: 0. -}; - -//Always success. -screen_stat_handle_t FS_create_handle(void); - -int FS_set_para(screen_stat_handle_t handle, enum FS_option type,const void* value,int size); -void FS_start(screen_stat_handle_t handle); -void FS_stop(screen_stat_handle_t* handle); - -//return field_id/line_id/column_id greater than zero if success,return an interger less than zero if failed. -//should NOT include "|:\n\r.\t<>[]#!@"or space in the parameter name. -//Runtime rregister column is NOT allowed. -int FS_register(screen_stat_handle_t handle,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name); - -//numerator_id and denominator_id must be column/field/status style. -//scaling: negative value: zoom in; positive value: zoom out; -int FS_register_ratio(screen_stat_handle_t handle,int numerator_id,int denominator_id,int scaling,enum field_dsp_style_t style,enum field_calc_algo calc_type,const char* name); - -inline void record_time_start(struct timespec *start) -{ - clock_gettime(CLOCK_MONOTONIC, start); -} -inline long record_time_elapse_us(struct timespec *start) -{ - struct timespec end; - long elapsed = 0; - clock_gettime(CLOCK_MONOTONIC, &end); - elapsed = (end.tv_sec - start->tv_sec)*1000000 + (end.tv_nsec - start->tv_nsec)/1000; - return elapsed; -} - -//@param bins format is comma spited number, e.g."0.1,0.5,0.8,0.9,0.95,0.99" -//return 0 on success, <0 on failed. -int FS_histogram_set_bins(screen_stat_handle_t handle, int id, const char* bins); - -//@param lowest_trackable_value >1 -//@param highest_trackable_value>lowest_trackable_value * 2 -//@param 1 - * Copyright (c) 2010-2014, Pieter Noordhuis - * Copyright (c) 2015, Matt Stancliff , - * Jan-Erik Rediger - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __HIREDIS_H -#define __HIREDIS_H -#include "read.h" -#include /* for va_list */ -#ifndef _MSC_VER -#include /* for struct timeval */ -#else -struct timeval; /* forward declaration */ -typedef long long ssize_t; -#endif -#include /* uintXX_t, etc */ -#include "sds.h" /* for sds */ -#include "alloc.h" /* for allocation wrappers */ - -#define HIREDIS_MAJOR 1 -#define HIREDIS_MINOR 0 -#define HIREDIS_PATCH 2 -#define HIREDIS_SONAME 1.0.0 - -/* Connection type can be blocking or non-blocking and is set in the - * least significant bit of the flags field in redisContext. */ -#define REDIS_BLOCK 0x1 - -/* Connection may be disconnected before being free'd. The second bit - * in the flags field is set when the context is connected. */ -#define REDIS_CONNECTED 0x2 - -/* The async API might try to disconnect cleanly and flush the output - * buffer and read all subsequent replies before disconnecting. - * This flag means no new commands can come in and the connection - * should be terminated once all replies have been read. */ -#define REDIS_DISCONNECTING 0x4 - -/* Flag specific to the async API which means that the context should be clean - * up as soon as possible. */ -#define REDIS_FREEING 0x8 - -/* Flag that is set when an async callback is executed. */ -#define REDIS_IN_CALLBACK 0x10 - -/* Flag that is set when the async context has one or more subscriptions. */ -#define REDIS_SUBSCRIBED 0x20 - -/* Flag that is set when monitor mode is active */ -#define REDIS_MONITORING 0x40 - -/* Flag that is set when we should set SO_REUSEADDR before calling bind() */ -#define REDIS_REUSEADDR 0x80 - -/** - * Flag that indicates the user does not want the context to - * be automatically freed upon error - */ -#define REDIS_NO_AUTO_FREE 0x200 - -#define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ - -/* number of times we retry to connect in the case of EADDRNOTAVAIL and - * SO_REUSEADDR is being used. */ -#define REDIS_CONNECT_RETRIES 10 - -/* Forward declarations for structs defined elsewhere */ -struct redisAsyncContext; -struct redisContext; - -/* RESP3 push helpers and callback prototypes */ -#define redisIsPushReply(r) (((redisReply*)(r))->type == REDIS_REPLY_PUSH) -typedef void (redisPushFn)(void *, void *); -typedef void (redisAsyncPushFn)(struct redisAsyncContext *, void *); - -#ifdef __cplusplus -extern "C" { -#endif - -/* This is the reply object returned by redisCommand() */ -typedef struct redisReply { - int type; /* REDIS_REPLY_* */ - long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ - double dval; /* The double when type is REDIS_REPLY_DOUBLE */ - size_t len; /* Length of string */ - char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING - REDIS_REPLY_VERB, and REDIS_REPLY_DOUBLE (in additional to dval). */ - char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null - terminated 3 character content type, such as "txt". */ - size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ - struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ -} redisReply; - -redisReader *redisReaderCreate(void); - -/* Function to free the reply objects hiredis returns by default. */ -void freeReplyObject(void *reply); - -/* Functions to format a command according to the protocol. */ -int redisvFormatCommand(char **target, const char *format, va_list ap); -int redisFormatCommand(char **target, const char *format, ...); -int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); -int redisFormatSdsCommandArgv(sds *target, int argc, const char ** argv, const size_t *argvlen); -void redisFreeCommand(char *cmd); -void redisFreeSdsCommand(sds cmd); - -enum redisConnectionType { - REDIS_CONN_TCP, - REDIS_CONN_UNIX, - REDIS_CONN_USERFD -}; - -struct redisSsl; - -#define REDIS_OPT_NONBLOCK 0x01 -#define REDIS_OPT_REUSEADDR 0x02 - -/** - * Don't automatically free the async object on a connection failure, - * or other implicit conditions. Only free on an explicit call to disconnect() or free() - */ -#define REDIS_OPT_NOAUTOFREE 0x04 - -/* Don't automatically intercept and free RESP3 PUSH replies. */ -#define REDIS_OPT_NO_PUSH_AUTOFREE 0x08 - -/* In Unix systems a file descriptor is a regular signed int, with -1 - * representing an invalid descriptor. In Windows it is a SOCKET - * (32- or 64-bit unsigned integer depending on the architecture), where - * all bits set (~0) is INVALID_SOCKET. */ -#ifndef _WIN32 -typedef int redisFD; -#define REDIS_INVALID_FD -1 -#else -#ifdef _WIN64 -typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ -#else -typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ -#endif -#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ -#endif - -typedef struct { - /* - * the type of connection to use. This also indicates which - * `endpoint` member field to use - */ - int type; - /* bit field of REDIS_OPT_xxx */ - int options; - /* timeout value for connect operation. If NULL, no timeout is used */ - const struct timeval *connect_timeout; - /* timeout value for commands. If NULL, no timeout is used. This can be - * updated at runtime with redisSetTimeout/redisAsyncSetTimeout. */ - const struct timeval *command_timeout; - union { - /** use this field for tcp/ip connections */ - struct { - const char *source_addr; - const char *ip; - int port; - } tcp; - /** use this field for unix domain sockets */ - const char *unix_socket; - /** - * use this field to have hiredis operate an already-open - * file descriptor */ - redisFD fd; - } endpoint; - - /* Optional user defined data/destructor */ - void *privdata; - void (*free_privdata)(void *); - - /* A user defined PUSH message callback */ - redisPushFn *push_cb; - redisAsyncPushFn *async_push_cb; -} redisOptions; - -/** - * Helper macros to initialize options to their specified fields. - */ -#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ - (opts)->type = REDIS_CONN_TCP; \ - (opts)->endpoint.tcp.ip = ip_; \ - (opts)->endpoint.tcp.port = port_; - -#define REDIS_OPTIONS_SET_UNIX(opts, path) \ - (opts)->type = REDIS_CONN_UNIX; \ - (opts)->endpoint.unix_socket = path; - -#define REDIS_OPTIONS_SET_PRIVDATA(opts, data, dtor) \ - (opts)->privdata = data; \ - (opts)->free_privdata = dtor; \ - -typedef struct redisContextFuncs { - void (*free_privctx)(void *); - void (*async_read)(struct redisAsyncContext *); - void (*async_write)(struct redisAsyncContext *); - ssize_t (*read)(struct redisContext *, char *, size_t); - ssize_t (*write)(struct redisContext *); -} redisContextFuncs; - -/* Context for a connection to Redis */ -typedef struct redisContext { - const redisContextFuncs *funcs; /* Function table */ - - int err; /* Error flags, 0 when there is no error */ - char errstr[128]; /* String representation of error when applicable */ - redisFD fd; - int flags; - char *obuf; /* Write buffer */ - redisReader *reader; /* Protocol reader */ - - enum redisConnectionType connection_type; - struct timeval *connect_timeout; - struct timeval *command_timeout; - - struct { - char *host; - char *source_addr; - int port; - } tcp; - - struct { - char *path; - } unix_sock; - - /* For non-blocking connect */ - struct sockadr *saddr; - size_t addrlen; - - /* Optional data and corresponding destructor users can use to provide - * context to a given redisContext. Not used by hiredis. */ - void *privdata; - void (*free_privdata)(void *); - - /* Internal context pointer presently used by hiredis to manage - * SSL connections. */ - void *privctx; - - /* An optional RESP3 PUSH handler */ - redisPushFn *push_cb; -} redisContext; - -redisContext *redisConnectWithOptions(const redisOptions *options); -redisContext *redisConnect(const char *ip, int port); -redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); -redisContext *redisConnectNonBlock(const char *ip, int port); -redisContext *redisConnectBindNonBlock(const char *ip, int port, - const char *source_addr); -redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, - const char *source_addr); -redisContext *redisConnectUnix(const char *path); -redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); -redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(redisFD fd); - -/** - * Reconnect the given context using the saved information. - * - * This re-uses the exact same connect options as in the initial connection. - * host, ip (or path), timeout and bind address are reused, - * flags are used unmodified from the existing context. - * - * Returns REDIS_OK on successful connect or REDIS_ERR otherwise. - */ -int redisReconnect(redisContext *c); - -redisPushFn *redisSetPushCallback(redisContext *c, redisPushFn *fn); -int redisSetTimeout(redisContext *c, const struct timeval tv); -int redisEnableKeepAlive(redisContext *c); -void redisFree(redisContext *c); -redisFD redisFreeKeepFd(redisContext *c); -int redisBufferRead(redisContext *c); -int redisBufferWrite(redisContext *c, int *done); - -/* In a blocking context, this function first checks if there are unconsumed - * replies to return and returns one if so. Otherwise, it flushes the output - * buffer to the socket and reads until it has a reply. In a non-blocking - * context, it will return unconsumed replies until there are no more. */ -int redisGetReply(redisContext *c, void **reply); -int redisGetReplyFromReader(redisContext *c, void **reply); - -/* Write a formatted command to the output buffer. Use these functions in blocking mode - * to get a pipeline of commands. */ -int redisAppendFormattedCommand(redisContext *c, const char *cmd, size_t len); - -/* Write a command to the output buffer. Use these functions in blocking mode - * to get a pipeline of commands. */ -int redisvAppendCommand(redisContext *c, const char *format, va_list ap); -int redisAppendCommand(redisContext *c, const char *format, ...); -int redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - -/* Issue a command to Redis. In a blocking context, it is identical to calling - * redisAppendCommand, followed by redisGetReply. The function will return - * NULL if there was an error in performing the request, otherwise it will - * return the reply. In a non-blocking context, it is identical to calling - * only redisAppendCommand and will always return NULL. */ -void *redisvCommand(redisContext *c, const char *format, va_list ap); -void *redisCommand(redisContext *c, const char *format, ...); -void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc_internal/view_only/hiredis/read.h b/src/inc_internal/view_only/hiredis/read.h deleted file mode 100644 index 2d74d77..0000000 --- a/src/inc_internal/view_only/hiredis/read.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2009-2011, Salvatore Sanfilippo - * Copyright (c) 2010-2011, Pieter Noordhuis - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - - -#ifndef __HIREDIS_READ_H -#define __HIREDIS_READ_H -#include /* for size_t */ - -#define REDIS_ERR -1 -#define REDIS_OK 0 - -/* When an error occurs, the err flag in a context is set to hold the type of - * error that occurred. REDIS_ERR_IO means there was an I/O error and you - * should use the "errno" variable to find out what is wrong. - * For other values, the "errstr" field will hold a description. */ -#define REDIS_ERR_IO 1 /* Error in read or write */ -#define REDIS_ERR_EOF 3 /* End of file */ -#define REDIS_ERR_PROTOCOL 4 /* Protocol error */ -#define REDIS_ERR_OOM 5 /* Out of memory */ -#define REDIS_ERR_TIMEOUT 6 /* Timed out */ -#define REDIS_ERR_OTHER 2 /* Everything else... */ - -#define REDIS_REPLY_STRING 1 -#define REDIS_REPLY_ARRAY 2 -#define REDIS_REPLY_INTEGER 3 -#define REDIS_REPLY_NIL 4 -#define REDIS_REPLY_STATUS 5 -#define REDIS_REPLY_ERROR 6 -#define REDIS_REPLY_DOUBLE 7 -#define REDIS_REPLY_BOOL 8 -#define REDIS_REPLY_MAP 9 -#define REDIS_REPLY_SET 10 -#define REDIS_REPLY_ATTR 11 -#define REDIS_REPLY_PUSH 12 -#define REDIS_REPLY_BIGNUM 13 -#define REDIS_REPLY_VERB 14 - -/* Default max unused reader buffer. */ -#define REDIS_READER_MAX_BUF (1024*16) - -/* Default multi-bulk element limit */ -#define REDIS_READER_MAX_ARRAY_ELEMENTS ((1LL<<32) - 1) - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct redisReadTask { - int type; - long long elements; /* number of elements in multibulk container */ - int idx; /* index in parent (array) object */ - void *obj; /* holds user-generated value for a read task */ - struct redisReadTask *parent; /* parent task */ - void *privdata; /* user-settable arbitrary field */ -} redisReadTask; - -typedef struct redisReplyObjectFunctions { - void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, size_t); - void *(*createInteger)(const redisReadTask*, long long); - void *(*createDouble)(const redisReadTask*, double, char*, size_t); - void *(*createNil)(const redisReadTask*); - void *(*createBool)(const redisReadTask*, int); - void (*freeObject)(void*); -} redisReplyObjectFunctions; - -typedef struct redisReader { - int err; /* Error flags, 0 when there is no error */ - char errstr[128]; /* String representation of error when applicable */ - - char *buf; /* Read buffer */ - size_t pos; /* Buffer cursor */ - size_t len; /* Buffer length */ - size_t maxbuf; /* Max length of unused buffer */ - long long maxelements; /* Max multi-bulk elements */ - - redisReadTask **task; - int tasks; - - int ridx; /* Index of current read task */ - void *reply; /* Temporary reply pointer */ - - redisReplyObjectFunctions *fn; - void *privdata; -} redisReader; - -/* Public API for the protocol parser. */ -redisReader *redisReaderCreateWithFunctions(redisReplyObjectFunctions *fn); -void redisReaderFree(redisReader *r); -int redisReaderFeed(redisReader *r, const char *buf, size_t len); -int redisReaderGetReply(redisReader *r, void **reply); - -#define redisReaderSetPrivdata(_r, _p) (int)(((redisReader*)(_r))->privdata = (_p)) -#define redisReaderGetObject(_r) (((redisReader*)(_r))->reply) -#define redisReaderGetError(_r) (((redisReader*)(_r))->errstr) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/inc_internal/view_only/hiredis/sds.h b/src/inc_internal/view_only/hiredis/sds.h deleted file mode 100644 index eda8833..0000000 --- a/src/inc_internal/view_only/hiredis/sds.h +++ /dev/null @@ -1,278 +0,0 @@ -/* SDSLib 2.0 -- A C dynamic strings library - * - * Copyright (c) 2006-2015, Salvatore Sanfilippo - * Copyright (c) 2015, Oran Agra - * Copyright (c) 2015, Redis Labs, Inc - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __SDS_H -#define __SDS_H - -#define SDS_MAX_PREALLOC (1024*1024) -#ifdef _MSC_VER -#define __attribute__(x) -typedef long long ssize_t; -#define SSIZE_MAX (LLONG_MAX >> 1) -#endif - -#include -#include -#include - -typedef char *sds; - -/* Note: sdshdr5 is never used, we just access the flags byte directly. - * However is here to document the layout of type 5 SDS strings. */ -struct __attribute__ ((__packed__)) sdshdr5 { - unsigned char flags; /* 3 lsb of type, and 5 msb of string length */ - char buf[]; -}; -struct __attribute__ ((__packed__)) sdshdr8 { - uint8_t len; /* used */ - uint8_t alloc; /* excluding the header and null terminator */ - unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; -}; -struct __attribute__ ((__packed__)) sdshdr16 { - uint16_t len; /* used */ - uint16_t alloc; /* excluding the header and null terminator */ - unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; -}; -struct __attribute__ ((__packed__)) sdshdr32 { - uint32_t len; /* used */ - uint32_t alloc; /* excluding the header and null terminator */ - unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; -}; -struct __attribute__ ((__packed__)) sdshdr64 { - uint64_t len; /* used */ - uint64_t alloc; /* excluding the header and null terminator */ - unsigned char flags; /* 3 lsb of type, 5 unused bits */ - char buf[]; -}; - -#define SDS_TYPE_5 0 -#define SDS_TYPE_8 1 -#define SDS_TYPE_16 2 -#define SDS_TYPE_32 3 -#define SDS_TYPE_64 4 -#define SDS_TYPE_MASK 7 -#define SDS_TYPE_BITS 3 -#define SDS_HDR_VAR(T,s) struct sdshdr##T *sh = (struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T))); -#define SDS_HDR(T,s) ((struct sdshdr##T *)((s)-(sizeof(struct sdshdr##T)))) -#define SDS_TYPE_5_LEN(f) ((f)>>SDS_TYPE_BITS) - -static inline size_t sdslen(const sds s) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->len; - case SDS_TYPE_16: - return SDS_HDR(16,s)->len; - case SDS_TYPE_32: - return SDS_HDR(32,s)->len; - case SDS_TYPE_64: - return SDS_HDR(64,s)->len; - } - return 0; -} - -static inline size_t sdsavail(const sds s) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: { - return 0; - } - case SDS_TYPE_8: { - SDS_HDR_VAR(8,s); - return sh->alloc - sh->len; - } - case SDS_TYPE_16: { - SDS_HDR_VAR(16,s); - return sh->alloc - sh->len; - } - case SDS_TYPE_32: { - SDS_HDR_VAR(32,s); - return sh->alloc - sh->len; - } - case SDS_TYPE_64: { - SDS_HDR_VAR(64,s); - return sh->alloc - sh->len; - } - } - return 0; -} - -static inline void sdssetlen(sds s, size_t newlen) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - { - unsigned char *fp = ((unsigned char*)s)-1; - *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); - } - break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len = (uint8_t)newlen; - break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len = (uint16_t)newlen; - break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len = (uint32_t)newlen; - break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len = (uint64_t)newlen; - break; - } -} - -static inline void sdsinclen(sds s, size_t inc) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - { - unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); - } - break; - case SDS_TYPE_8: - SDS_HDR(8,s)->len += (uint8_t)inc; - break; - case SDS_TYPE_16: - SDS_HDR(16,s)->len += (uint16_t)inc; - break; - case SDS_TYPE_32: - SDS_HDR(32,s)->len += (uint32_t)inc; - break; - case SDS_TYPE_64: - SDS_HDR(64,s)->len += (uint64_t)inc; - break; - } -} - -/* sdsalloc() = sdsavail() + sdslen() */ -static inline size_t sdsalloc(const sds s) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - return SDS_TYPE_5_LEN(flags); - case SDS_TYPE_8: - return SDS_HDR(8,s)->alloc; - case SDS_TYPE_16: - return SDS_HDR(16,s)->alloc; - case SDS_TYPE_32: - return SDS_HDR(32,s)->alloc; - case SDS_TYPE_64: - return SDS_HDR(64,s)->alloc; - } - return 0; -} - -static inline void sdssetalloc(sds s, size_t newlen) { - unsigned char flags = s[-1]; - switch(flags&SDS_TYPE_MASK) { - case SDS_TYPE_5: - /* Nothing to do, this type has no total allocation info. */ - break; - case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = (uint8_t)newlen; - break; - case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = (uint16_t)newlen; - break; - case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = (uint32_t)newlen; - break; - case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = (uint64_t)newlen; - break; - } -} - -sds sdsnewlen(const void *init, size_t initlen); -sds sdsnew(const char *init); -sds sdsempty(void); -sds sdsdup(const sds s); -void sdsfree(sds s); -sds sdsgrowzero(sds s, size_t len); -sds sdscatlen(sds s, const void *t, size_t len); -sds sdscat(sds s, const char *t); -sds sdscatsds(sds s, const sds t); -sds sdscpylen(sds s, const char *t, size_t len); -sds sdscpy(sds s, const char *t); - -sds sdscatvprintf(sds s, const char *fmt, va_list ap); -#ifdef __GNUC__ -sds sdscatprintf(sds s, const char *fmt, ...) - __attribute__((format(printf, 2, 3))); -#else -sds sdscatprintf(sds s, const char *fmt, ...); -#endif - -sds sdscatfmt(sds s, char const *fmt, ...); -sds sdstrim(sds s, const char *cset); -int sdsrange(sds s, ssize_t start, ssize_t end); -void sdsupdatelen(sds s); -void sdsclear(sds s); -int sdscmp(const sds s1, const sds s2); -sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); -void sdsfreesplitres(sds *tokens, int count); -void sdstolower(sds s); -void sdstoupper(sds s); -sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, const char *p, size_t len); -sds *sdssplitargs(const char *line, int *argc); -sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); -sds sdsjoin(char **argv, int argc, char *sep); -sds sdsjoinsds(sds *argv, int argc, const char *sep, size_t seplen); - -/* Low level functions exposed to the user API */ -sds sdsMakeRoomFor(sds s, size_t addlen); -void sdsIncrLen(sds s, int incr); -sds sdsRemoveFreeSpace(sds s); -size_t sdsAllocSize(sds s); -void *sdsAllocPtr(sds s); - -/* Export the allocator used by SDS to the program using SDS. - * Sometimes the program SDS is linked to, may use a different set of - * allocators, but may want to allocate or free things that SDS will - * respectively free or allocate. */ -void *sds_malloc(size_t size); -void *sds_realloc(void *ptr, size_t size); -void sds_free(void *ptr); - -#ifdef REDIS_TEST -int sdsTest(int argc, char *argv[]); -#endif - -#endif diff --git a/src/inc_internal/view_only/hyperscan/hs.h b/src/inc_internal/view_only/hyperscan/hs.h deleted file mode 100644 index 2fe5d24..0000000 --- a/src/inc_internal/view_only/hyperscan/hs.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HS_H_ -#define HS_H_ - -/** - * @file - * @brief The complete Hyperscan API definition. - * - * Hyperscan is a high speed regular expression engine. - * - * This header includes both the Hyperscan compiler and runtime components. See - * the individual component headers for documentation. - */ - -/* The current Hyperscan version information. */ - -#define HS_MAJOR 5 -#define HS_MINOR 4 -#define HS_PATCH 0 - -#include "hs_compile.h" -#include "hs_runtime.h" - -#endif /* HS_H_ */ diff --git a/src/inc_internal/view_only/hyperscan/hs_common.h b/src/inc_internal/view_only/hyperscan/hs_common.h deleted file mode 100644 index 93dc1fe..0000000 --- a/src/inc_internal/view_only/hyperscan/hs_common.h +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2015-2019, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HS_COMMON_H_ -#define HS_COMMON_H_ - -#if defined(_WIN32) -#define HS_CDECL __cdecl -#else -#define HS_CDECL -#endif -#include - -/** - * @file - * @brief The Hyperscan common API definition. - * - * Hyperscan is a high speed regular expression engine. - * - * This header contains functions available to both the Hyperscan compiler and - * runtime. - */ - -#ifdef __cplusplus -extern "C" -{ -#endif - -struct hs_database; - -/** - * A Hyperscan pattern database. - * - * Generated by one of the Hyperscan compiler functions: - * - @ref hs_compile() - * - @ref hs_compile_multi() - * - @ref hs_compile_ext_multi() - */ -typedef struct hs_database hs_database_t; - -/** - * A type for errors returned by Hyperscan functions. - */ -typedef int hs_error_t; - -/** - * Free a compiled pattern database. - * - * The free callback set by @ref hs_set_database_allocator() (or @ref - * hs_set_allocator()) will be used by this function. - * - * @param db - * A compiled pattern database. NULL may also be safely provided, in which - * case the function does nothing. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_free_database(hs_database_t *db); - -/** - * Serialize a pattern database to a stream of bytes. - * - * The allocator callback set by @ref hs_set_misc_allocator() (or @ref - * hs_set_allocator()) will be used by this function. - * - * @param db - * A compiled pattern database. - * - * @param bytes - * On success, a pointer to an array of bytes will be returned here. - * These bytes can be subsequently relocated or written to disk. The - * caller is responsible for freeing this block. - * - * @param length - * On success, the number of bytes in the generated byte array will be - * returned here. - * - * @return - * @ref HS_SUCCESS on success, @ref HS_NOMEM if the byte array cannot be - * allocated, other values may be returned if errors are detected. - */ -hs_error_t HS_CDECL hs_serialize_database(const hs_database_t *db, char **bytes, - size_t *length); - -/** - * Reconstruct a pattern database from a stream of bytes previously generated - * by @ref hs_serialize_database(). - * - * This function will allocate sufficient space for the database using the - * allocator set with @ref hs_set_database_allocator() (or @ref - * hs_set_allocator()); to use a pre-allocated region of memory, use the @ref - * hs_deserialize_database_at() function. - * - * @param bytes - * A byte array generated by @ref hs_serialize_database() representing a - * compiled pattern database. - * - * @param length - * The length of the byte array generated by @ref hs_serialize_database(). - * This should be the same value as that returned by @ref - * hs_serialize_database(). - * - * @param db - * On success, a pointer to a newly allocated @ref hs_database_t will be - * returned here. This database can then be used for scanning, and - * eventually freed by the caller using @ref hs_free_database(). - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_deserialize_database(const char *bytes, - const size_t length, - hs_database_t **db); - -/** - * Reconstruct a pattern database from a stream of bytes previously generated - * by @ref hs_serialize_database() at a given memory location. - * - * This function (unlike @ref hs_deserialize_database()) will write the - * reconstructed database to the memory location given in the @p db parameter. - * The amount of space required at this location can be determined with the - * @ref hs_serialized_database_size() function. - * - * @param bytes - * A byte array generated by @ref hs_serialize_database() representing a - * compiled pattern database. - * - * @param length - * The length of the byte array generated by @ref hs_serialize_database(). - * This should be the same value as that returned by @ref - * hs_serialize_database(). - * - * @param db - * Pointer to an 8-byte aligned block of memory of sufficient size to hold - * the deserialized database. On success, the reconstructed database will - * be written to this location. This database can then be used for pattern - * matching. The user is responsible for freeing this memory; the @ref - * hs_free_database() call should not be used. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_deserialize_database_at(const char *bytes, - const size_t length, - hs_database_t *db); - -/** - * Provides the size of the stream state allocated by a single stream opened - * against the given database. - * - * @param database - * Pointer to a compiled (streaming mode) pattern database. - * - * @param stream_size - * On success, the size in bytes of an individual stream opened against the - * given database is placed in this parameter. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_stream_size(const hs_database_t *database, - size_t *stream_size); - -/** - * Provides the size of the given database in bytes. - * - * @param database - * Pointer to compiled pattern database. - * - * @param database_size - * On success, the size of the compiled database in bytes is placed in this - * parameter. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_database_size(const hs_database_t *database, - size_t *database_size); - -/** - * Utility function for reporting the size that would be required by a - * database if it were deserialized. - * - * This can be used to allocate a shared memory region or other "special" - * allocation prior to deserializing with the @ref hs_deserialize_database_at() - * function. - * - * @param bytes - * Pointer to a byte array generated by @ref hs_serialize_database() - * representing a compiled pattern database. - * - * @param length - * The length of the byte array generated by @ref hs_serialize_database(). - * This should be the same value as that returned by @ref - * hs_serialize_database(). - * - * @param deserialized_size - * On success, the size of the compiled database that would be generated - * by @ref hs_deserialize_database_at() is returned here. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_serialized_database_size(const char *bytes, - const size_t length, - size_t *deserialized_size); - -/** - * Utility function providing information about a database. - * - * @param database - * Pointer to a compiled database. - * - * @param info - * On success, a string containing the version and platform information for - * the supplied database is placed in the parameter. The string is - * allocated using the allocator supplied in @ref hs_set_misc_allocator() - * (or malloc() if no allocator was set) and should be freed by the caller. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_database_info(const hs_database_t *database, - char **info); - -/** - * Utility function providing information about a serialized database. - * - * @param bytes - * Pointer to a serialized database. - * - * @param length - * Length in bytes of the serialized database. - * - * @param info - * On success, a string containing the version and platform information - * for the supplied serialized database is placed in the parameter. The - * string is allocated using the allocator supplied in @ref - * hs_set_misc_allocator() (or malloc() if no allocator was set) and - * should be freed by the caller. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_serialized_database_info(const char *bytes, - size_t length, char **info); - -/** - * The type of the callback function that will be used by Hyperscan to allocate - * more memory at runtime as required, for example in @ref hs_open_stream() to - * allocate stream state. - * - * If Hyperscan is to be used in a multi-threaded, or similarly concurrent - * environment, the allocation function will need to be re-entrant, or - * similarly safe for concurrent use. - * - * @param size - * The number of bytes to allocate. - * @return - * A pointer to the region of memory allocated, or NULL on error. - */ -typedef void *(HS_CDECL *hs_alloc_t)(size_t size); - -/** - * The type of the callback function that will be used by Hyperscan to free - * memory regions previously allocated using the @ref hs_alloc_t function. - * - * @param ptr - * The region of memory to be freed. - */ -typedef void (HS_CDECL *hs_free_t)(void *ptr); - -/** - * Set the allocate and free functions used by Hyperscan for allocating - * memory at runtime for stream state, scratch space, database bytecode, - * and various other data structure returned by the Hyperscan API. - * - * The function is equivalent to calling @ref hs_set_stream_allocator(), - * @ref hs_set_scratch_allocator(), @ref hs_set_database_allocator() and - * @ref hs_set_misc_allocator() with the provided parameters. - * - * This call will override any previous allocators that have been set. - * - * Note: there is no way to change the allocator used for temporary objects - * created during the various compile calls (@ref hs_compile(), @ref - * hs_compile_multi(), @ref hs_compile_ext_multi()). - * - * @param alloc_func - * A callback function pointer that allocates memory. This function must - * return memory suitably aligned for the largest representable data type - * on this platform. - * - * @param free_func - * A callback function pointer that frees allocated memory. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_set_allocator(hs_alloc_t alloc_func, - hs_free_t free_func); - -/** - * Set the allocate and free functions used by Hyperscan for allocating memory - * for database bytecode produced by the compile calls (@ref hs_compile(), @ref - * hs_compile_multi(), @ref hs_compile_ext_multi()) and by database - * deserialization (@ref hs_deserialize_database()). - * - * If no database allocation functions are set, or if NULL is used in place of - * both parameters, then memory allocation will default to standard methods - * (such as the system malloc() and free() calls). - * - * This call will override any previous database allocators that have been set. - * - * Note: the database allocator may also be set by calling @ref - * hs_set_allocator(). - * - * Note: there is no way to change how temporary objects created during the - * various compile calls (@ref hs_compile(), @ref hs_compile_multi(), @ref - * hs_compile_ext_multi()) are allocated. - * - * @param alloc_func - * A callback function pointer that allocates memory. This function must - * return memory suitably aligned for the largest representable data type - * on this platform. - * - * @param free_func - * A callback function pointer that frees allocated memory. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_set_database_allocator(hs_alloc_t alloc_func, - hs_free_t free_func); - -/** - * Set the allocate and free functions used by Hyperscan for allocating memory - * for items returned by the Hyperscan API such as @ref hs_compile_error_t, @ref - * hs_expr_info_t and serialized databases. - * - * If no misc allocation functions are set, or if NULL is used in place of both - * parameters, then memory allocation will default to standard methods (such as - * the system malloc() and free() calls). - * - * This call will override any previous misc allocators that have been set. - * - * Note: the misc allocator may also be set by calling @ref hs_set_allocator(). - * - * @param alloc_func - * A callback function pointer that allocates memory. This function must - * return memory suitably aligned for the largest representable data type - * on this platform. - * - * @param free_func - * A callback function pointer that frees allocated memory. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_set_misc_allocator(hs_alloc_t alloc_func, - hs_free_t free_func); - -/** - * Set the allocate and free functions used by Hyperscan for allocating memory - * for scratch space by @ref hs_alloc_scratch() and @ref hs_clone_scratch(). - * - * If no scratch allocation functions are set, or if NULL is used in place of - * both parameters, then memory allocation will default to standard methods - * (such as the system malloc() and free() calls). - * - * This call will override any previous scratch allocators that have been set. - * - * Note: the scratch allocator may also be set by calling @ref - * hs_set_allocator(). - * - * @param alloc_func - * A callback function pointer that allocates memory. This function must - * return memory suitably aligned for the largest representable data type - * on this platform. - * - * @param free_func - * A callback function pointer that frees allocated memory. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_set_scratch_allocator(hs_alloc_t alloc_func, - hs_free_t free_func); - -/** - * Set the allocate and free functions used by Hyperscan for allocating memory - * for stream state by @ref hs_open_stream(). - * - * If no stream allocation functions are set, or if NULL is used in place of - * both parameters, then memory allocation will default to standard methods - * (such as the system malloc() and free() calls). - * - * This call will override any previous stream allocators that have been set. - * - * Note: the stream allocator may also be set by calling @ref - * hs_set_allocator(). - * - * @param alloc_func - * A callback function pointer that allocates memory. This function must - * return memory suitably aligned for the largest representable data type - * on this platform. - * - * @param free_func - * A callback function pointer that frees allocated memory. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_set_stream_allocator(hs_alloc_t alloc_func, - hs_free_t free_func); - -/** - * Utility function for identifying this release version. - * - * @return - * A string containing the version number of this release build and the - * date of the build. It is allocated statically, so it does not need to - * be freed by the caller. - */ -const char * HS_CDECL hs_version(void); - -/** - * Utility function to test the current system architecture. - * - * Hyperscan requires the Supplemental Streaming SIMD Extensions 3 instruction - * set. This function can be called on any x86 platform to determine if the - * system provides the required instruction set. - * - * This function does not test for more advanced features if Hyperscan has - * been built for a more specific architecture, for example the AVX2 - * instruction set. - * - * @return - * @ref HS_SUCCESS on success, @ref HS_ARCH_ERROR if system does not - * support Hyperscan. - */ -hs_error_t HS_CDECL hs_valid_platform(void); - -/** - * @defgroup HS_ERROR hs_error_t values - * - * @{ - */ - -/** - * The engine completed normally. - */ -#define HS_SUCCESS 0 - -/** - * A parameter passed to this function was invalid. - * - * This error is only returned in cases where the function can detect an - * invalid parameter -- it cannot be relied upon to detect (for example) - * pointers to freed memory or other invalid data. - */ -#define HS_INVALID (-1) - -/** - * A memory allocation failed. - */ -#define HS_NOMEM (-2) - -/** - * The engine was terminated by callback. - * - * This return value indicates that the target buffer was partially scanned, - * but that the callback function requested that scanning cease after a match - * was located. - */ -#define HS_SCAN_TERMINATED (-3) - -/** - * The pattern compiler failed, and the @ref hs_compile_error_t should be - * inspected for more detail. - */ -#define HS_COMPILER_ERROR (-4) - -/** - * The given database was built for a different version of Hyperscan. - */ -#define HS_DB_VERSION_ERROR (-5) - -/** - * The given database was built for a different platform (i.e., CPU type). - */ -#define HS_DB_PLATFORM_ERROR (-6) - -/** - * The given database was built for a different mode of operation. This error - * is returned when streaming calls are used with a block or vectored database - * and vice versa. - */ -#define HS_DB_MODE_ERROR (-7) - -/** - * A parameter passed to this function was not correctly aligned. - */ -#define HS_BAD_ALIGN (-8) - -/** - * The memory allocator (either malloc() or the allocator set with @ref - * hs_set_allocator()) did not correctly return memory suitably aligned for the - * largest representable data type on this platform. - */ -#define HS_BAD_ALLOC (-9) - -/** - * The scratch region was already in use. - * - * This error is returned when Hyperscan is able to detect that the scratch - * region given is already in use by another Hyperscan API call. - * - * A separate scratch region, allocated with @ref hs_alloc_scratch() or @ref - * hs_clone_scratch(), is required for every concurrent caller of the Hyperscan - * API. - * - * For example, this error might be returned when @ref hs_scan() has been - * called inside a callback delivered by a currently-executing @ref hs_scan() - * call using the same scratch region. - * - * Note: Not all concurrent uses of scratch regions may be detected. This error - * is intended as a best-effort debugging tool, not a guarantee. - */ -#define HS_SCRATCH_IN_USE (-10) - -/** - * Unsupported CPU architecture. - * - * This error is returned when Hyperscan is able to detect that the current - * system does not support the required instruction set. - * - * At a minimum, Hyperscan requires Supplemental Streaming SIMD Extensions 3 - * (SSSE3). - */ -#define HS_ARCH_ERROR (-11) - -/** - * Provided buffer was too small. - * - * This error indicates that there was insufficient space in the buffer. The - * call should be repeated with a larger provided buffer. - * - * Note: in this situation, it is normal for the amount of space required to be - * returned in the same manner as the used space would have been returned if the - * call was successful. - */ -#define HS_INSUFFICIENT_SPACE (-12) - -/** - * Unexpected internal error. - * - * This error indicates that there was unexpected matching behaviors. This - * could be related to invalid usage of stream and scratch space or invalid memory - * operations by users. - * - */ -#define HS_UNKNOWN_ERROR (-13) - -/** @} */ - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* HS_COMMON_H_ */ diff --git a/src/inc_internal/view_only/hyperscan/hs_compile.h b/src/inc_internal/view_only/hyperscan/hs_compile.h deleted file mode 100644 index b318c29..0000000 --- a/src/inc_internal/view_only/hyperscan/hs_compile.h +++ /dev/null @@ -1,1224 +0,0 @@ -/* - * Copyright (c) 2015-2020, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HS_COMPILE_H_ -#define HS_COMPILE_H_ - -/** - * @file - * @brief The Hyperscan compiler API definition. - * - * Hyperscan is a high speed regular expression engine. - * - * This header contains functions for compiling regular expressions into - * Hyperscan databases that can be used by the Hyperscan runtime. - */ - -#include "hs_common.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - -/** - * A type containing error details that is returned by the compile calls (@ref - * hs_compile(), @ref hs_compile_multi() and @ref hs_compile_ext_multi()) on - * failure. The caller may inspect the values returned in this type to - * determine the cause of failure. - * - * Common errors generated during the compile process include: - * - * - *Invalid parameter* - * - * An invalid argument was specified in the compile call. - * - * - *Unrecognised flag* - * - * An unrecognised value was passed in the flags argument. - * - * - *Pattern matches empty buffer* - * - * By default, Hyperscan only supports patterns that will *always* - * consume at least one byte of input. Patterns that do not have this - * property (such as `/(abc)?/`) will produce this error unless - * the @ref HS_FLAG_ALLOWEMPTY flag is supplied. Note that such - * patterns will produce a match for *every* byte when scanned. - * - * - *Embedded anchors not supported* - * - * Hyperscan only supports the use of anchor meta-characters (such as - * `^` and `$`) in patterns where they could *only* match - * at the start or end of a buffer. A pattern containing an embedded - * anchor, such as `/abc^def/`, can never match, as there is no - * way for `abc` to precede the start of the data stream. - * - * - *Bounded repeat is too large* - * - * The pattern contains a repeated construct with very large finite - * bounds. - * - * - *Unsupported component type* - * - * An unsupported PCRE construct was used in the pattern. - * - * - *Unable to generate bytecode* - * - * This error indicates that Hyperscan was unable to compile a pattern - * that is syntactically valid. The most common cause is a pattern that is - * very long and complex or contains a large repeated subpattern. - * - * - *Unable to allocate memory* - * - * The library was unable to allocate temporary storage used during - * compilation time. - * - * - *Allocator returned misaligned memory* - * - * The memory allocator (either malloc() or the allocator set with @ref - * hs_set_allocator()) did not correctly return memory suitably aligned - * for the largest representable data type on this platform. - * - * - *Internal error* - * - * An unexpected error occurred: if this error is reported, please contact - * the Hyperscan team with a description of the situation. - */ -typedef struct hs_compile_error { - /** - * A human-readable error message describing the error. - */ - char *message; - - /** - * The zero-based number of the expression that caused the error (if this - * can be determined). If the error is not specific to an expression, then - * this value will be less than zero. - */ - int expression; -} hs_compile_error_t; - -/** - * A type containing information on the target platform which may optionally be - * provided to the compile calls (@ref hs_compile(), @ref hs_compile_multi(), - * @ref hs_compile_ext_multi()). - * - * A hs_platform_info structure may be populated for the current platform by - * using the @ref hs_populate_platform() call. - */ -typedef struct hs_platform_info { - /** - * Information about the target platform which may be used to guide the - * optimisation process of the compile. - * - * Use of this field does not limit the processors that the resulting - * database can run on, but may impact the performance of the resulting - * database. - */ - unsigned int tune; - - /** - * Relevant CPU features available on the target platform - * - * This value may be produced by combining HS_CPU_FEATURE_* flags (such as - * @ref HS_CPU_FEATURES_AVX2). Multiple CPU features may be or'ed together - * to produce the value. - */ - unsigned long long cpu_features; - - /** - * Reserved for future use. - */ - unsigned long long reserved1; - - /** - * Reserved for future use. - */ - unsigned long long reserved2; -} hs_platform_info_t; - -/** - * A type containing information related to an expression that is returned by - * @ref hs_expression_info() or @ref hs_expression_ext_info. - */ -typedef struct hs_expr_info { - /** - * The minimum length in bytes of a match for the pattern. - * - * Note: in some cases when using advanced features to suppress matches - * (such as extended parameters or the @ref HS_FLAG_SINGLEMATCH flag) this - * may represent a conservative lower bound for the true minimum length of - * a match. - */ - unsigned int min_width; - - /** - * The maximum length in bytes of a match for the pattern. If the pattern - * has an unbounded maximum length, this will be set to the maximum value - * of an unsigned int (UINT_MAX). - * - * Note: in some cases when using advanced features to suppress matches - * (such as extended parameters or the @ref HS_FLAG_SINGLEMATCH flag) this - * may represent a conservative upper bound for the true maximum length of - * a match. - */ - unsigned int max_width; - - /** - * Whether this expression can produce matches that are not returned in - * order, such as those produced by assertions. Zero if false, non-zero if - * true. - */ - char unordered_matches; - - /** - * Whether this expression can produce matches at end of data (EOD). In - * streaming mode, EOD matches are raised during @ref hs_close_stream(), - * since it is only when @ref hs_close_stream() is called that the EOD - * location is known. Zero if false, non-zero if true. - * - * Note: trailing `\b` word boundary assertions may also result in EOD - * matches as end-of-data can act as a word boundary. - */ - char matches_at_eod; - - /** - * Whether this expression can *only* produce matches at end of data (EOD). - * In streaming mode, all matches for this expression are raised during - * @ref hs_close_stream(). Zero if false, non-zero if true. - */ - char matches_only_at_eod; -} hs_expr_info_t; - -/** - * A structure containing additional parameters related to an expression, - * passed in at build time to @ref hs_compile_ext_multi() or @ref - * hs_expression_ext_info. - * - * These parameters allow the set of matches produced by a pattern to be - * constrained at compile time, rather than relying on the application to - * process unwanted matches at runtime. - */ -typedef struct hs_expr_ext { - /** - * Flags governing which parts of this structure are to be used by the - * compiler. See @ref HS_EXT_FLAG. - */ - unsigned long long flags; - - /** - * The minimum end offset in the data stream at which this expression - * should match successfully. To use this parameter, set the - * @ref HS_EXT_FLAG_MIN_OFFSET flag in the hs_expr_ext::flags field. - */ - unsigned long long min_offset; - - /** - * The maximum end offset in the data stream at which this expression - * should match successfully. To use this parameter, set the - * @ref HS_EXT_FLAG_MAX_OFFSET flag in the hs_expr_ext::flags field. - */ - unsigned long long max_offset; - - /** - * The minimum match length (from start to end) required to successfully - * match this expression. To use this parameter, set the - * @ref HS_EXT_FLAG_MIN_LENGTH flag in the hs_expr_ext::flags field. - */ - unsigned long long min_length; - - /** - * Allow patterns to approximately match within this edit distance. To use - * this parameter, set the @ref HS_EXT_FLAG_EDIT_DISTANCE flag in the - * hs_expr_ext::flags field. - */ - unsigned edit_distance; - - /** - * Allow patterns to approximately match within this Hamming distance. To - * use this parameter, set the @ref HS_EXT_FLAG_HAMMING_DISTANCE flag in the - * hs_expr_ext::flags field. - */ - unsigned hamming_distance; -} hs_expr_ext_t; - -/** - * @defgroup HS_EXT_FLAG hs_expr_ext_t flags - * - * These flags are used in @ref hs_expr_ext_t::flags to indicate which fields - * are used. - * - * @{ - */ - -/** Flag indicating that the hs_expr_ext::min_offset field is used. */ -#define HS_EXT_FLAG_MIN_OFFSET 1ULL - -/** Flag indicating that the hs_expr_ext::max_offset field is used. */ -#define HS_EXT_FLAG_MAX_OFFSET 2ULL - -/** Flag indicating that the hs_expr_ext::min_length field is used. */ -#define HS_EXT_FLAG_MIN_LENGTH 4ULL - -/** Flag indicating that the hs_expr_ext::edit_distance field is used. */ -#define HS_EXT_FLAG_EDIT_DISTANCE 8ULL - -/** Flag indicating that the hs_expr_ext::hamming_distance field is used. */ -#define HS_EXT_FLAG_HAMMING_DISTANCE 16ULL - -/** @} */ - -/** - * The basic regular expression compiler. - * - * This is the function call with which an expression is compiled into a - * Hyperscan database which can be passed to the runtime functions (such as - * @ref hs_scan(), @ref hs_open_stream(), etc.) - * - * @param expression - * The NULL-terminated expression to parse. Note that this string must - * represent ONLY the pattern to be matched, with no delimiters or flags; - * any global flags should be specified with the @p flags argument. For - * example, the expression `/abc?def/i` should be compiled by providing - * `abc?def` as the @p expression, and @ref HS_FLAG_CASELESS as the @a - * flags. - * - * @param flags - * Flags which modify the behaviour of the expression. Multiple flags may - * be used by ORing them together. Valid values are: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_DOTALL - Matching a `.` will not exclude newlines. - * - HS_FLAG_MULTILINE - `^` and `$` anchors match any newlines in data. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated for the - * expression per stream. - * - HS_FLAG_ALLOWEMPTY - Allow expressions which can match against an - * empty string, such as `.*`. - * - HS_FLAG_UTF8 - Treat this pattern as a sequence of UTF-8 characters. - * - HS_FLAG_UCP - Use Unicode properties for character classes. - * - HS_FLAG_PREFILTER - Compile pattern in prefiltering mode. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - HS_FLAG_COMBINATION - Parse the expression in logical combination - * syntax. - * - HS_FLAG_QUIET - Ignore match reporting for this expression. Used for - * the sub-expressions in logical combinations. - * - * @param mode - * Compiler mode flags that affect the database as a whole. One of @ref - * HS_MODE_STREAM or @ref HS_MODE_BLOCK or @ref HS_MODE_VECTORED must be - * supplied, to select between the generation of a streaming, block or - * vectored database. In addition, other flags (beginning with HS_MODE_) - * may be supplied to enable specific features. See @ref HS_MODE_FLAG for - * more details. - * - * @param platform - * If not NULL, the platform structure is used to determine the target - * platform for the database. If NULL, a database suitable for running - * on the current host platform is produced. - * - * @param db - * On success, a pointer to the generated database will be returned in - * this parameter, or NULL on failure. The caller is responsible for - * deallocating the buffer using the @ref hs_free_database() function. - * - * @param error - * If the compile fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the error - * parameter. - */ -hs_error_t HS_CDECL hs_compile(const char *expression, unsigned int flags, - unsigned int mode, - const hs_platform_info_t *platform, - hs_database_t **db, hs_compile_error_t **error); - -/** - * The multiple regular expression compiler. - * - * This is the function call with which a set of expressions is compiled into a - * database which can be passed to the runtime functions (such as @ref - * hs_scan(), @ref hs_open_stream(), etc.) Each expression can be labelled with - * a unique integer which is passed into the match callback to identify the - * pattern that has matched. - * - * @param expressions - * Array of NULL-terminated expressions to compile. Note that (as for @ref - * hs_compile()) these strings must contain only the pattern to be - * matched, with no delimiters or flags. For example, the expression - * `/abc?def/i` should be compiled by providing `abc?def` as the first - * string in the @p expressions array, and @ref HS_FLAG_CASELESS as the - * first value in the @p flags array. - * - * @param flags - * Array of flags which modify the behaviour of each expression. Multiple - * flags may be used by ORing them together. Specifying the NULL pointer - * in place of an array will set the flags value for all patterns to zero. - * Valid values are: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_DOTALL - Matching a `.` will not exclude newlines. - * - HS_FLAG_MULTILINE - `^` and `$` anchors match any newlines in data. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated by patterns - * with this match id per stream. - * - HS_FLAG_ALLOWEMPTY - Allow expressions which can match against an - * empty string, such as `.*`. - * - HS_FLAG_UTF8 - Treat this pattern as a sequence of UTF-8 characters. - * - HS_FLAG_UCP - Use Unicode properties for character classes. - * - HS_FLAG_PREFILTER - Compile pattern in prefiltering mode. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - HS_FLAG_COMBINATION - Parse the expression in logical combination - * syntax. - * - HS_FLAG_QUIET - Ignore match reporting for this expression. Used for - * the sub-expressions in logical combinations. - * - * @param ids - * An array of integers specifying the ID number to be associated with the - * corresponding pattern in the expressions array. Specifying the NULL - * pointer in place of an array will set the ID value for all patterns to - * zero. - * - * @param elements - * The number of elements in the input arrays. - * - * @param mode - * Compiler mode flags that affect the database as a whole. One of @ref - * HS_MODE_STREAM or @ref HS_MODE_BLOCK or @ref HS_MODE_VECTORED must be - * supplied, to select between the generation of a streaming, block or - * vectored database. In addition, other flags (beginning with HS_MODE_) - * may be supplied to enable specific features. See @ref HS_MODE_FLAG for - * more details. - * - * @param platform - * If not NULL, the platform structure is used to determine the target - * platform for the database. If NULL, a database suitable for running - * on the current host platform is produced. - * - * @param db - * On success, a pointer to the generated database will be returned in - * this parameter, or NULL on failure. The caller is responsible for - * deallocating the buffer using the @ref hs_free_database() function. - * - * @param error - * If the compile fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the @p error - * parameter. - * - */ -hs_error_t HS_CDECL hs_compile_multi(const char *const *expressions, - const unsigned int *flags, - const unsigned int *ids, - unsigned int elements, unsigned int mode, - const hs_platform_info_t *platform, - hs_database_t **db, - hs_compile_error_t **error); - -/** - * The multiple regular expression compiler with extended parameter support. - * - * This function call compiles a group of expressions into a database in the - * same way as @ref hs_compile_multi(), but allows additional parameters to be - * specified via an @ref hs_expr_ext_t structure per expression. - * - * @param expressions - * Array of NULL-terminated expressions to compile. Note that (as for @ref - * hs_compile()) these strings must contain only the pattern to be - * matched, with no delimiters or flags. For example, the expression - * `/abc?def/i` should be compiled by providing `abc?def` as the first - * string in the @p expressions array, and @ref HS_FLAG_CASELESS as the - * first value in the @p flags array. - * - * @param flags - * Array of flags which modify the behaviour of each expression. Multiple - * flags may be used by ORing them together. Specifying the NULL pointer - * in place of an array will set the flags value for all patterns to zero. - * Valid values are: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_DOTALL - Matching a `.` will not exclude newlines. - * - HS_FLAG_MULTILINE - `^` and `$` anchors match any newlines in data. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated by patterns - * with this match id per stream. - * - HS_FLAG_ALLOWEMPTY - Allow expressions which can match against an - * empty string, such as `.*`. - * - HS_FLAG_UTF8 - Treat this pattern as a sequence of UTF-8 characters. - * - HS_FLAG_UCP - Use Unicode properties for character classes. - * - HS_FLAG_PREFILTER - Compile pattern in prefiltering mode. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - HS_FLAG_COMBINATION - Parse the expression in logical combination - * syntax. - * - HS_FLAG_QUIET - Ignore match reporting for this expression. Used for - * the sub-expressions in logical combinations. - * - * @param ids - * An array of integers specifying the ID number to be associated with the - * corresponding pattern in the expressions array. Specifying the NULL - * pointer in place of an array will set the ID value for all patterns to - * zero. - * - * @param ext - * An array of pointers to filled @ref hs_expr_ext_t structures that - * define extended behaviour for each pattern. NULL may be specified if no - * extended behaviour is needed for an individual pattern, or in place of - * the whole array if it is not needed for any expressions. Memory used by - * these structures must be both allocated and freed by the caller. - * - * @param elements - * The number of elements in the input arrays. - * - * @param mode - * Compiler mode flags that affect the database as a whole. One of @ref - * HS_MODE_STREAM, @ref HS_MODE_BLOCK or @ref HS_MODE_VECTORED must be - * supplied, to select between the generation of a streaming, block or - * vectored database. In addition, other flags (beginning with HS_MODE_) - * may be supplied to enable specific features. See @ref HS_MODE_FLAG for - * more details. - * - * @param platform - * If not NULL, the platform structure is used to determine the target - * platform for the database. If NULL, a database suitable for running - * on the current host platform is produced. - * - * @param db - * On success, a pointer to the generated database will be returned in - * this parameter, or NULL on failure. The caller is responsible for - * deallocating the buffer using the @ref hs_free_database() function. - * - * @param error - * If the compile fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the @p error - * parameter. - * - */ -hs_error_t HS_CDECL hs_compile_ext_multi(const char *const *expressions, - const unsigned int *flags, - const unsigned int *ids, - const hs_expr_ext_t *const *ext, - unsigned int elements, unsigned int mode, - const hs_platform_info_t *platform, - hs_database_t **db, hs_compile_error_t **error); - -/** - * The basic pure literal expression compiler. - * - * This is the function call with which a pure literal expression (not a - * common regular expression) is compiled into a Hyperscan database which - * can be passed to the runtime functions (such as @ref hs_scan(), - * @ref hs_open_stream(), etc.) - * - * @param expression - * The NULL-terminated expression to parse. Note that this string must - * represent ONLY the pattern to be matched, with no delimiters or flags; - * any global flags should be specified with the @p flags argument. For - * example, the expression `/abc?def/i` should be compiled by providing - * `abc?def` as the @p expression, and @ref HS_FLAG_CASELESS as the @a - * flags. Meanwhile, the string content shall be fully parsed in a literal - * sense without any regular grammars. For example, the @p expression - * `abc?` simply means a char sequence of `a`, `b`, `c`, and `?`. The `?` - * here doesn't mean 0 or 1 quantifier under regular semantics. - * - * @param flags - * Flags which modify the behaviour of the expression. Multiple flags may - * be used by ORing them together. Compared to @ref hs_compile(), fewer - * valid values are provided: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated for the - * expression per stream. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - * @param len - * The length of the text content of the pure literal expression. As the - * text content indicated by @p expression is treated as single character - * one by one, the special terminating character `\0` should be allowed - * to appear in expression, and not treated as a terminator for a string. - * Thus, the end of a pure literal expression cannot be indicated by - * identifying `\0`, but by counting to the expression length. - * - * @param mode - * Compiler mode flags that affect the database as a whole. One of @ref - * HS_MODE_STREAM or @ref HS_MODE_BLOCK or @ref HS_MODE_VECTORED must be - * supplied, to select between the generation of a streaming, block or - * vectored database. In addition, other flags (beginning with HS_MODE_) - * may be supplied to enable specific features. See @ref HS_MODE_FLAG for - * more details. - * - * @param platform - * If not NULL, the platform structure is used to determine the target - * platform for the database. If NULL, a database suitable for running - * on the current host platform is produced. - * - * @param db - * On success, a pointer to the generated database will be returned in - * this parameter, or NULL on failure. The caller is responsible for - * deallocating the buffer using the @ref hs_free_database() function. - * - * @param error - * If the compile fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the error - * parameter. - */ -hs_error_t HS_CDECL hs_compile_lit(const char *expression, unsigned flags, - const size_t len, unsigned mode, - const hs_platform_info_t *platform, - hs_database_t **db, - hs_compile_error_t **error); -/** - * The multiple pure literal expression compiler. - * - * This is the function call with which a set of pure literal expressions is - * compiled into a database which can be passed to the runtime functions (such - * as @ref hs_scan(), @ref hs_open_stream(), etc.) Each expression can be - * labelled with a unique integer which is passed into the match callback to - * identify the pattern that has matched. - * - * @param expressions - * The NULL-terminated expression to parse. Note that this string must - * represent ONLY the pattern to be matched, with no delimiters or flags; - * any global flags should be specified with the @p flags argument. For - * example, the expression `/abc?def/i` should be compiled by providing - * `abc?def` as the @p expression, and @ref HS_FLAG_CASELESS as the @a - * flags. Meanwhile, the string content shall be fully parsed in a literal - * sense without any regular grammars. For example, the @p expression - * `abc?` simply means a char sequence of `a`, `b`, `c`, and `?`. The `?` - * here doesn't mean 0 or 1 quantifier under regular semantics. - * - * @param flags - * Array of flags which modify the behaviour of each expression. Multiple - * flags may be used by ORing them together. Specifying the NULL pointer - * in place of an array will set the flags value for all patterns to zero. - * Compared to @ref hs_compile_multi(), fewer valid values are provided: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated for the - * expression per stream. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - * @param ids - * An array of integers specifying the ID number to be associated with the - * corresponding pattern in the expressions array. Specifying the NULL - * pointer in place of an array will set the ID value for all patterns to - * zero. - * - * @param lens - * Array of lengths of the text content of each pure literal expression. - * As the text content indicated by @p expression is treated as single - * character one by one, the special terminating character `\0` should be - * allowed to appear in expression, and not treated as a terminator for a - * string. Thus, the end of a pure literal expression cannot be indicated - * by identifying `\0`, but by counting to the expression length. - * - * @param elements - * The number of elements in the input arrays. - * - * @param mode - * Compiler mode flags that affect the database as a whole. One of @ref - * HS_MODE_STREAM or @ref HS_MODE_BLOCK or @ref HS_MODE_VECTORED must be - * supplied, to select between the generation of a streaming, block or - * vectored database. In addition, other flags (beginning with HS_MODE_) - * may be supplied to enable specific features. See @ref HS_MODE_FLAG for - * more details. - * - * @param platform - * If not NULL, the platform structure is used to determine the target - * platform for the database. If NULL, a database suitable for running - * on the current host platform is produced. - * - * @param db - * On success, a pointer to the generated database will be returned in - * this parameter, or NULL on failure. The caller is responsible for - * deallocating the buffer using the @ref hs_free_database() function. - * - * @param error - * If the compile fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the error - * parameter. - */ -hs_error_t HS_CDECL hs_compile_lit_multi(const char * const *expressions, - const unsigned *flags, - const unsigned *ids, - const size_t *lens, - unsigned elements, unsigned mode, - const hs_platform_info_t *platform, - hs_database_t **db, - hs_compile_error_t **error); - -/** - * Free an error structure generated by @ref hs_compile(), @ref - * hs_compile_multi() or @ref hs_compile_ext_multi(). - * - * @param error - * The @ref hs_compile_error_t to be freed. NULL may also be safely - * provided. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_free_compile_error(hs_compile_error_t *error); - -/** - * Utility function providing information about a regular expression. The - * information provided in @ref hs_expr_info_t includes the minimum and maximum - * width of a pattern match. - * - * Note: successful analysis of an expression with this function does not imply - * that compilation of the same expression (via @ref hs_compile(), @ref - * hs_compile_multi() or @ref hs_compile_ext_multi()) would succeed. This - * function may return @ref HS_SUCCESS for regular expressions that Hyperscan - * cannot compile. - * - * Note: some per-pattern flags (such as @ref HS_FLAG_ALLOWEMPTY, @ref - * HS_FLAG_SOM_LEFTMOST) are accepted by this call, but as they do not affect - * the properties returned in the @ref hs_expr_info_t structure, they will not - * affect the outcome of this function. - * - * @param expression - * The NULL-terminated expression to parse. Note that this string must - * represent ONLY the pattern to be matched, with no delimiters or flags; - * any global flags should be specified with the @p flags argument. For - * example, the expression `/abc?def/i` should be compiled by providing - * `abc?def` as the @p expression, and @ref HS_FLAG_CASELESS as the @a - * flags. - * - * @param flags - * Flags which modify the behaviour of the expression. Multiple flags may - * be used by ORing them together. Valid values are: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_DOTALL - Matching a `.` will not exclude newlines. - * - HS_FLAG_MULTILINE - `^` and `$` anchors match any newlines in data. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated by the - * expression per stream. - * - HS_FLAG_ALLOWEMPTY - Allow expressions which can match against an - * empty string, such as `.*`. - * - HS_FLAG_UTF8 - Treat this pattern as a sequence of UTF-8 characters. - * - HS_FLAG_UCP - Use Unicode properties for character classes. - * - HS_FLAG_PREFILTER - Compile pattern in prefiltering mode. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - HS_FLAG_COMBINATION - Parse the expression in logical combination - * syntax. - * - HS_FLAG_QUIET - Ignore match reporting for this expression. Used for - * the sub-expressions in logical combinations. - * - * @param info - * On success, a pointer to the pattern information will be returned in - * this parameter, or NULL on failure. This structure is allocated using - * the allocator supplied in @ref hs_set_allocator() (or malloc() if no - * allocator was set) and should be freed by the caller. - * - * @param error - * If the call fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the error - * parameter. - */ -hs_error_t HS_CDECL hs_expression_info(const char *expression, - unsigned int flags, - hs_expr_info_t **info, - hs_compile_error_t **error); - -/** - * Utility function providing information about a regular expression, with - * extended parameter support. The information provided in @ref hs_expr_info_t - * includes the minimum and maximum width of a pattern match. - * - * Note: successful analysis of an expression with this function does not imply - * that compilation of the same expression (via @ref hs_compile(), @ref - * hs_compile_multi() or @ref hs_compile_ext_multi()) would succeed. This - * function may return @ref HS_SUCCESS for regular expressions that Hyperscan - * cannot compile. - * - * Note: some per-pattern flags (such as @ref HS_FLAG_ALLOWEMPTY, @ref - * HS_FLAG_SOM_LEFTMOST) are accepted by this call, but as they do not affect - * the properties returned in the @ref hs_expr_info_t structure, they will not - * affect the outcome of this function. - * - * @param expression - * The NULL-terminated expression to parse. Note that this string must - * represent ONLY the pattern to be matched, with no delimiters or flags; - * any global flags should be specified with the @p flags argument. For - * example, the expression `/abc?def/i` should be compiled by providing - * `abc?def` as the @p expression, and @ref HS_FLAG_CASELESS as the @a - * flags. - * - * @param flags - * Flags which modify the behaviour of the expression. Multiple flags may - * be used by ORing them together. Valid values are: - * - HS_FLAG_CASELESS - Matching will be performed case-insensitively. - * - HS_FLAG_DOTALL - Matching a `.` will not exclude newlines. - * - HS_FLAG_MULTILINE - `^` and `$` anchors match any newlines in data. - * - HS_FLAG_SINGLEMATCH - Only one match will be generated by the - * expression per stream. - * - HS_FLAG_ALLOWEMPTY - Allow expressions which can match against an - * empty string, such as `.*`. - * - HS_FLAG_UTF8 - Treat this pattern as a sequence of UTF-8 characters. - * - HS_FLAG_UCP - Use Unicode properties for character classes. - * - HS_FLAG_PREFILTER - Compile pattern in prefiltering mode. - * - HS_FLAG_SOM_LEFTMOST - Report the leftmost start of match offset - * when a match is found. - * - HS_FLAG_COMBINATION - Parse the expression in logical combination - * syntax. - * - HS_FLAG_QUIET - Ignore match reporting for this expression. Used for - * the sub-expressions in logical combinations. - * - * @param ext - * A pointer to a filled @ref hs_expr_ext_t structure that defines - * extended behaviour for this pattern. NULL may be specified if no - * extended parameters are needed. - * - * @param info - * On success, a pointer to the pattern information will be returned in - * this parameter, or NULL on failure. This structure is allocated using - * the allocator supplied in @ref hs_set_allocator() (or malloc() if no - * allocator was set) and should be freed by the caller. - * - * @param error - * If the call fails, a pointer to a @ref hs_compile_error_t will be - * returned, providing details of the error condition. The caller is - * responsible for deallocating the buffer using the @ref - * hs_free_compile_error() function. - * - * @return - * @ref HS_SUCCESS is returned on successful compilation; @ref - * HS_COMPILER_ERROR on failure, with details provided in the error - * parameter. - */ -hs_error_t HS_CDECL hs_expression_ext_info(const char *expression, - unsigned int flags, - const hs_expr_ext_t *ext, - hs_expr_info_t **info, - hs_compile_error_t **error); - -/** - * Populates the platform information based on the current host. - * - * @param platform - * On success, the pointed to structure is populated based on the current - * host. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_populate_platform(hs_platform_info_t *platform); - -/** - * @defgroup HS_PATTERN_FLAG Pattern flags - * - * @{ - */ - -/** - * Compile flag: Set case-insensitive matching. - * - * This flag sets the expression to be matched case-insensitively by default. - * The expression may still use PCRE tokens (notably `(?i)` and - * `(?-i)`) to switch case-insensitive matching on and off. - */ -#define HS_FLAG_CASELESS 1 - -/** - * Compile flag: Matching a `.` will not exclude newlines. - * - * This flag sets any instances of the `.` token to match newline characters as - * well as all other characters. The PCRE specification states that the `.` - * token does not match newline characters by default, so without this flag the - * `.` token will not cross line boundaries. - */ -#define HS_FLAG_DOTALL 2 - -/** - * Compile flag: Set multi-line anchoring. - * - * This flag instructs the expression to make the `^` and `$` tokens match - * newline characters as well as the start and end of the stream. If this flag - * is not specified, the `^` token will only ever match at the start of a - * stream, and the `$` token will only ever match at the end of a stream within - * the guidelines of the PCRE specification. - */ -#define HS_FLAG_MULTILINE 4 - -/** - * Compile flag: Set single-match only mode. - * - * This flag sets the expression's match ID to match at most once. In streaming - * mode, this means that the expression will return only a single match over - * the lifetime of the stream, rather than reporting every match as per - * standard Hyperscan semantics. In block mode or vectored mode, only the first - * match for each invocation of @ref hs_scan() or @ref hs_scan_vector() will be - * returned. - * - * If multiple expressions in the database share the same match ID, then they - * either must all specify @ref HS_FLAG_SINGLEMATCH or none of them specify - * @ref HS_FLAG_SINGLEMATCH. If a group of expressions sharing a match ID - * specify the flag, then at most one match with the match ID will be generated - * per stream. - * - * Note: The use of this flag in combination with @ref HS_FLAG_SOM_LEFTMOST - * is not currently supported. - */ -#define HS_FLAG_SINGLEMATCH 8 - -/** - * Compile flag: Allow expressions that can match against empty buffers. - * - * This flag instructs the compiler to allow expressions that can match against - * empty buffers, such as `.?`, `.*`, `(a|)`. Since Hyperscan can return every - * possible match for an expression, such expressions generally execute very - * slowly; the default behaviour is to return an error when an attempt to - * compile one is made. Using this flag will force the compiler to allow such - * an expression. - */ -#define HS_FLAG_ALLOWEMPTY 16 - -/** - * Compile flag: Enable UTF-8 mode for this expression. - * - * This flag instructs Hyperscan to treat the pattern as a sequence of UTF-8 - * characters. The results of scanning invalid UTF-8 sequences with a Hyperscan - * library that has been compiled with one or more patterns using this flag are - * undefined. - */ -#define HS_FLAG_UTF8 32 - -/** - * Compile flag: Enable Unicode property support for this expression. - * - * This flag instructs Hyperscan to use Unicode properties, rather than the - * default ASCII interpretations, for character mnemonics like `\w` and `\s` as - * well as the POSIX character classes. It is only meaningful in conjunction - * with @ref HS_FLAG_UTF8. - */ -#define HS_FLAG_UCP 64 - -/** - * Compile flag: Enable prefiltering mode for this expression. - * - * This flag instructs Hyperscan to compile an "approximate" version of this - * pattern for use in a prefiltering application, even if Hyperscan does not - * support the pattern in normal operation. - * - * The set of matches returned when this flag is used is guaranteed to be a - * superset of the matches specified by the non-prefiltering expression. - * - * If the pattern contains pattern constructs not supported by Hyperscan (such - * as zero-width assertions, back-references or conditional references) these - * constructs will be replaced internally with broader constructs that may - * match more often. - * - * Furthermore, in prefiltering mode Hyperscan may simplify a pattern that - * would otherwise return a "Pattern too large" error at compile time, or for - * performance reasons (subject to the matching guarantee above). - * - * It is generally expected that the application will subsequently confirm - * prefilter matches with another regular expression matcher that can provide - * exact matches for the pattern. - * - * Note: The use of this flag in combination with @ref HS_FLAG_SOM_LEFTMOST - * is not currently supported. - */ -#define HS_FLAG_PREFILTER 128 - -/** - * Compile flag: Enable leftmost start of match reporting. - * - * This flag instructs Hyperscan to report the leftmost possible start of match - * offset when a match is reported for this expression. (By default, no start - * of match is returned.) - * - * For all the 3 modes, enabling this behaviour may reduce performance. And - * particularly, it may increase stream state requirements in streaming mode. - */ -#define HS_FLAG_SOM_LEFTMOST 256 - -/** - * Compile flag: Logical combination. - * - * This flag instructs Hyperscan to parse this expression as logical - * combination syntax. - * Logical constraints consist of operands, operators and parentheses. - * The operands are expression indices, and operators can be - * '!'(NOT), '&'(AND) or '|'(OR). - * For example: - * (101&102&103)|(104&!105) - * ((301|302)&303)&(304|305) - */ -#define HS_FLAG_COMBINATION 512 - -/** - * Compile flag: Don't do any match reporting. - * - * This flag instructs Hyperscan to ignore match reporting for this expression. - * It is designed to be used on the sub-expressions in logical combinations. - */ -#define HS_FLAG_QUIET 1024 - -/** @} */ - -/** - * @defgroup HS_CPU_FEATURES_FLAG CPU feature support flags - * - * @{ - */ - -/** - * CPU features flag - Intel(R) Advanced Vector Extensions 2 (Intel(R) AVX2) - * - * Setting this flag indicates that the target platform supports AVX2 - * instructions. - */ -#define HS_CPU_FEATURES_AVX2 (1ULL << 2) - -/** - * CPU features flag - Intel(R) Advanced Vector Extensions 512 (Intel(R) AVX512) - * - * Setting this flag indicates that the target platform supports AVX512 - * instructions, specifically AVX-512BW. Using AVX512 implies the use of AVX2. - */ -#define HS_CPU_FEATURES_AVX512 (1ULL << 3) - -/** - * CPU features flag - Intel(R) Advanced Vector Extensions 512 - * Vector Byte Manipulation Instructions (Intel(R) AVX512VBMI) - * - * Setting this flag indicates that the target platform supports AVX512VBMI - * instructions. Using AVX512VBMI implies the use of AVX512. - */ -#define HS_CPU_FEATURES_AVX512VBMI (1ULL << 4) - -/** @} */ - -/** - * @defgroup HS_TUNE_FLAG Tuning flags - * - * @{ - */ - -/** - * Tuning Parameter - Generic - * - * This indicates that the compiled database should not be tuned for any - * particular target platform. - */ -#define HS_TUNE_FAMILY_GENERIC 0 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Sandy Bridge - * - * This indicates that the compiled database should be tuned for the - * Sandy Bridge microarchitecture. - */ -#define HS_TUNE_FAMILY_SNB 1 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Ivy Bridge - * - * This indicates that the compiled database should be tuned for the - * Ivy Bridge microarchitecture. - */ -#define HS_TUNE_FAMILY_IVB 2 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Haswell - * - * This indicates that the compiled database should be tuned for the - * Haswell microarchitecture. - */ -#define HS_TUNE_FAMILY_HSW 3 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Silvermont - * - * This indicates that the compiled database should be tuned for the - * Silvermont microarchitecture. - */ -#define HS_TUNE_FAMILY_SLM 4 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Broadwell - * - * This indicates that the compiled database should be tuned for the - * Broadwell microarchitecture. - */ -#define HS_TUNE_FAMILY_BDW 5 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Skylake - * - * This indicates that the compiled database should be tuned for the - * Skylake microarchitecture. - */ -#define HS_TUNE_FAMILY_SKL 6 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Skylake Server - * - * This indicates that the compiled database should be tuned for the - * Skylake Server microarchitecture. - */ -#define HS_TUNE_FAMILY_SKX 7 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Goldmont - * - * This indicates that the compiled database should be tuned for the - * Goldmont microarchitecture. - */ -#define HS_TUNE_FAMILY_GLM 8 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Icelake - * - * This indicates that the compiled database should be tuned for the - * Icelake microarchitecture. - */ -#define HS_TUNE_FAMILY_ICL 9 - -/** - * Tuning Parameter - Intel(R) microarchitecture code name Icelake Server - * - * This indicates that the compiled database should be tuned for the - * Icelake Server microarchitecture. - */ -#define HS_TUNE_FAMILY_ICX 10 - -/** @} */ - -/** - * @defgroup HS_MODE_FLAG Compile mode flags - * - * The mode flags are used as values for the mode parameter of the various - * compile calls (@ref hs_compile(), @ref hs_compile_multi() and @ref - * hs_compile_ext_multi()). - * - * A mode value can be built by ORing these flag values together; the only - * required flag is one of @ref HS_MODE_BLOCK, @ref HS_MODE_STREAM or @ref - * HS_MODE_VECTORED. Other flags may be added to enable support for additional - * features. - * - * @{ - */ - -/** - * Compiler mode flag: Block scan (non-streaming) database. - */ -#define HS_MODE_BLOCK 1 - -/** - * Compiler mode flag: Alias for @ref HS_MODE_BLOCK. - */ -#define HS_MODE_NOSTREAM 1 - -/** - * Compiler mode flag: Streaming database. - */ -#define HS_MODE_STREAM 2 - -/** - * Compiler mode flag: Vectored scanning database. - */ -#define HS_MODE_VECTORED 4 - -/** - * Compiler mode flag: use full precision to track start of match offsets in - * stream state. - * - * This mode will use the most stream state per pattern, but will always return - * an accurate start of match offset regardless of how far back in the past it - * was found. - * - * One of the SOM_HORIZON modes must be selected to use the @ref - * HS_FLAG_SOM_LEFTMOST expression flag. - */ -#define HS_MODE_SOM_HORIZON_LARGE (1U << 24) - -/** - * Compiler mode flag: use medium precision to track start of match offsets in - * stream state. - * - * This mode will use less stream state than @ref HS_MODE_SOM_HORIZON_LARGE and - * will limit start of match accuracy to offsets within 2^32 bytes of the - * end of match offset reported. - * - * One of the SOM_HORIZON modes must be selected to use the @ref - * HS_FLAG_SOM_LEFTMOST expression flag. - */ -#define HS_MODE_SOM_HORIZON_MEDIUM (1U << 25) - -/** - * Compiler mode flag: use limited precision to track start of match offsets in - * stream state. - * - * This mode will use less stream state than @ref HS_MODE_SOM_HORIZON_LARGE and - * will limit start of match accuracy to offsets within 2^16 bytes of the - * end of match offset reported. - * - * One of the SOM_HORIZON modes must be selected to use the @ref - * HS_FLAG_SOM_LEFTMOST expression flag. - */ -#define HS_MODE_SOM_HORIZON_SMALL (1U << 26) - -/** @} */ - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* HS_COMPILE_H_ */ diff --git a/src/inc_internal/view_only/hyperscan/hs_runtime.h b/src/inc_internal/view_only/hyperscan/hs_runtime.h deleted file mode 100644 index 6d34b6c..0000000 --- a/src/inc_internal/view_only/hyperscan/hs_runtime.h +++ /dev/null @@ -1,621 +0,0 @@ -/* - * Copyright (c) 2015-2018, Intel Corporation - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Intel Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef HS_RUNTIME_H_ -#define HS_RUNTIME_H_ - -#include - -/** - * @file - * @brief The Hyperscan runtime API definition. - * - * Hyperscan is a high speed regular expression engine. - * - * This header contains functions for using compiled Hyperscan databases for - * scanning data at runtime. - */ - -#include "hs_common.h" - -#ifdef __cplusplus -extern "C" -{ -#endif - -/** - * Definition of the stream identifier type. - */ -struct hs_stream; - -/** - * The stream identifier returned by @ref hs_open_stream(). - */ -typedef struct hs_stream hs_stream_t; - -struct hs_scratch; - -/** - * A Hyperscan scratch space. - */ -typedef struct hs_scratch hs_scratch_t; - -/** - * Definition of the match event callback function type. - * - * A callback function matching the defined type must be provided by the - * application calling the @ref hs_scan(), @ref hs_scan_vector() or @ref - * hs_scan_stream() functions (or other streaming calls which can produce - * matches). - * - * This callback function will be invoked whenever a match is located in the - * target data during the execution of a scan. The details of the match are - * passed in as parameters to the callback function, and the callback function - * should return a value indicating whether or not matching should continue on - * the target data. If no callbacks are desired from a scan call, NULL may be - * provided in order to suppress match production. - * - * This callback function should not attempt to call Hyperscan API functions on - * the same stream nor should it attempt to reuse the scratch space allocated - * for the API calls that caused it to be triggered. Making another call to the - * Hyperscan library with completely independent parameters should work (for - * example, scanning a different database in a new stream and with new scratch - * space), but reusing data structures like stream state and/or scratch space - * will produce undefined behavior. - * - * @param id - * The ID number of the expression that matched. If the expression was a - * single expression compiled with @ref hs_compile(), this value will be - * zero. - * - * @param from - * - If a start of match flag is enabled for the current pattern, this - * argument will be set to the start of match for the pattern assuming - * that that start of match value lies within the current 'start of match - * horizon' chosen by one of the SOM_HORIZON mode flags. - - * - If the start of match value lies outside this horizon (possible only - * when the SOM_HORIZON value is not @ref HS_MODE_SOM_HORIZON_LARGE), - * the @p from value will be set to @ref HS_OFFSET_PAST_HORIZON. - - * - This argument will be set to zero if the Start of Match flag is not - * enabled for the given pattern. - * - * @param to - * The offset after the last byte that matches the expression. - * - * @param flags - * This is provided for future use and is unused at present. - * - * @param context - * The pointer supplied by the user to the @ref hs_scan(), @ref - * hs_scan_vector() or @ref hs_scan_stream() function. - * - * @return - * Non-zero if the matching should cease, else zero. If scanning is - * performed in streaming mode and a non-zero value is returned, any - * subsequent calls to @ref hs_scan_stream() for that stream will - * immediately return with @ref HS_SCAN_TERMINATED. - */ -typedef int (HS_CDECL *match_event_handler)(unsigned int id, - unsigned long long from, - unsigned long long to, - unsigned int flags, - void *context); - -/** - * Open and initialise a stream. - * - * @param db - * A compiled pattern database. - * - * @param flags - * Flags modifying the behaviour of the stream. This parameter is provided - * for future use and is unused at present. - * - * @param stream - * On success, a pointer to the generated @ref hs_stream_t will be - * returned; NULL on failure. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_open_stream(const hs_database_t *db, unsigned int flags, - hs_stream_t **stream); - -/** - * Write data to be scanned to the opened stream. - * - * This is the function call in which the actual pattern matching takes place - * as data is written to the stream. Matches will be returned via the @ref - * match_event_handler callback supplied. - * - * @param id - * The stream ID (returned by @ref hs_open_stream()) to which the data - * will be written. - * - * @param data - * Pointer to the data to be scanned. - * - * @param length - * The number of bytes to scan. - * - * @param flags - * Flags modifying the behaviour of the stream. This parameter is provided - * for future use and is unused at present. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch(). - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param ctxt - * The user defined pointer which will be passed to the callback function - * when a match occurs. - * - * @return - * Returns @ref HS_SUCCESS on success; @ref HS_SCAN_TERMINATED if the - * match callback indicated that scanning should stop; other values on - * error. - */ -hs_error_t HS_CDECL hs_scan_stream(hs_stream_t *id, const char *data, - unsigned int length, unsigned int flags, - hs_scratch_t *scratch, - match_event_handler onEvent, void *ctxt); - -/** - * Close a stream. - * - * This function completes matching on the given stream and frees the memory - * associated with the stream state. After this call, the stream pointed to by - * @p id is invalid and can no longer be used. To reuse the stream state after - * completion, rather than closing it, the @ref hs_reset_stream function can be - * used. - * - * This function must be called for any stream created with @ref - * hs_open_stream(), even if scanning has been terminated by a non-zero return - * from the match callback function. - * - * Note: This operation may result in matches being returned (via calls to the - * match event callback) for expressions anchored to the end of the data stream - * (for example, via the use of the `$` meta-character). If these matches are - * not desired, NULL may be provided as the @ref match_event_handler callback. - * - * If NULL is provided as the @ref match_event_handler callback, it is - * permissible to provide a NULL scratch. - * - * @param id - * The stream ID returned by @ref hs_open_stream(). - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch(). This is - * allowed to be NULL only if the @p onEvent callback is also NULL. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param ctxt - * The user defined pointer which will be passed to the callback function - * when a match occurs. - * - * @return - * Returns @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_close_stream(hs_stream_t *id, hs_scratch_t *scratch, - match_event_handler onEvent, void *ctxt); - -/** - * Reset a stream to an initial state. - * - * Conceptually, this is equivalent to performing @ref hs_close_stream() on the - * given stream, followed by a @ref hs_open_stream(). This new stream replaces - * the original stream in memory, avoiding the overhead of freeing the old - * stream and allocating the new one. - * - * Note: This operation may result in matches being returned (via calls to the - * match event callback) for expressions anchored to the end of the original - * data stream (for example, via the use of the `$` meta-character). If these - * matches are not desired, NULL may be provided as the @ref match_event_handler - * callback. - * - * Note: the stream will also be tied to the same database. - * - * @param id - * The stream (as created by @ref hs_open_stream()) to be replaced. - * - * @param flags - * Flags modifying the behaviour of the stream. This parameter is provided - * for future use and is unused at present. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch(). This is - * allowed to be NULL only if the @p onEvent callback is also NULL. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param context - * The user defined pointer which will be passed to the callback function - * when a match occurs. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_reset_stream(hs_stream_t *id, unsigned int flags, - hs_scratch_t *scratch, - match_event_handler onEvent, void *context); - -/** - * Duplicate the given stream. The new stream will have the same state as the - * original including the current stream offset. - * - * @param to_id - * On success, a pointer to the new, copied @ref hs_stream_t will be - * returned; NULL on failure. - * - * @param from_id - * The stream (as created by @ref hs_open_stream()) to be copied. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_copy_stream(hs_stream_t **to_id, - const hs_stream_t *from_id); - -/** - * Duplicate the given 'from' stream state onto the 'to' stream. The 'to' stream - * will first be reset (reporting any EOD matches if a non-NULL @p onEvent - * callback handler is provided). - * - * Note: the 'to' stream and the 'from' stream must be open against the same - * database. - * - * @param to_id - * On success, a pointer to the new, copied @ref hs_stream_t will be - * returned; NULL on failure. - * - * @param from_id - * The stream (as created by @ref hs_open_stream()) to be copied. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch(). This is - * allowed to be NULL only if the @p onEvent callback is also NULL. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param context - * The user defined pointer which will be passed to the callback function - * when a match occurs. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_reset_and_copy_stream(hs_stream_t *to_id, - const hs_stream_t *from_id, - hs_scratch_t *scratch, - match_event_handler onEvent, - void *context); - -/** - * Creates a compressed representation of the provided stream in the buffer - * provided. This compressed representation can be converted back into a stream - * state by using @ref hs_expand_stream() or @ref hs_reset_and_expand_stream(). - * The size of the compressed representation will be placed into @p used_space. - * - * If there is not sufficient space in the buffer to hold the compressed - * representation, @ref HS_INSUFFICIENT_SPACE will be returned and @p used_space - * will be populated with the amount of space required. - * - * Note: this function does not close the provided stream, you may continue to - * use the stream or to free it with @ref hs_close_stream(). - * - * @param stream - * The stream (as created by @ref hs_open_stream()) to be compressed. - * - * @param buf - * Buffer to write the compressed representation into. Note: if the call is - * just being used to determine the amount of space required, it is allowed - * to pass NULL here and @p buf_space as 0. - * - * @param buf_space - * The number of bytes in @p buf. If buf_space is too small, the call will - * fail with @ref HS_INSUFFICIENT_SPACE. - * - * @param used_space - * Pointer to where the amount of used space will be written to. The used - * buffer space is always less than or equal to @p buf_space. If the call - * fails with @ref HS_INSUFFICIENT_SPACE, this pointer will be used to - * write out the amount of buffer space required. - * - * @return - * @ref HS_SUCCESS on success, @ref HS_INSUFFICIENT_SPACE if the provided - * buffer is too small. - */ -hs_error_t HS_CDECL hs_compress_stream(const hs_stream_t *stream, char *buf, - size_t buf_space, size_t *used_space); - -/** - * Decompresses a compressed representation created by @ref hs_compress_stream() - * into a new stream. - * - * Note: @p buf must correspond to a complete compressed representation created - * by @ref hs_compress_stream() of a stream that was opened against @p db. It is - * not always possible to detect misuse of this API and behaviour is undefined - * if these properties are not satisfied. - * - * @param db - * The compiled pattern database that the compressed stream was opened - * against. - * - * @param stream - * On success, a pointer to the expanded @ref hs_stream_t will be - * returned; NULL on failure. - * - * @param buf - * A compressed representation of a stream. These compressed forms are - * created by @ref hs_compress_stream(). - * - * @param buf_size - * The size in bytes of the compressed representation. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_expand_stream(const hs_database_t *db, - hs_stream_t **stream, const char *buf, - size_t buf_size); - -/** - * Decompresses a compressed representation created by @ref hs_compress_stream() - * on top of the 'to' stream. The 'to' stream will first be reset (reporting - * any EOD matches if a non-NULL @p onEvent callback handler is provided). - * - * Note: the 'to' stream must be opened against the same database as the - * compressed stream. - * - * Note: @p buf must correspond to a complete compressed representation created - * by @ref hs_compress_stream() of a stream that was opened against @p db. It is - * not always possible to detect misuse of this API and behaviour is undefined - * if these properties are not satisfied. - * - * @param to_stream - * A pointer to a valid stream state. A pointer to the expanded @ref - * hs_stream_t will be returned; NULL on failure. - * - * @param buf - * A compressed representation of a stream. These compressed forms are - * created by @ref hs_compress_stream(). - * - * @param buf_size - * The size in bytes of the compressed representation. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch(). This is - * allowed to be NULL only if the @p onEvent callback is also NULL. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param context - * The user defined pointer which will be passed to the callback function - * when a match occurs. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_reset_and_expand_stream(hs_stream_t *to_stream, - const char *buf, size_t buf_size, - hs_scratch_t *scratch, - match_event_handler onEvent, - void *context); - -/** - * The block (non-streaming) regular expression scanner. - * - * This is the function call in which the actual pattern matching takes place - * for block-mode pattern databases. - * - * @param db - * A compiled pattern database. - * - * @param data - * Pointer to the data to be scanned. - * - * @param length - * The number of bytes to scan. - * - * @param flags - * Flags modifying the behaviour of this function. This parameter is - * provided for future use and is unused at present. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch() for this - * database. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param context - * The user defined pointer which will be passed to the callback function. - * - * @return - * Returns @ref HS_SUCCESS on success; @ref HS_SCAN_TERMINATED if the - * match callback indicated that scanning should stop; other values on - * error. - */ -hs_error_t HS_CDECL hs_scan(const hs_database_t *db, const char *data, - unsigned int length, unsigned int flags, - hs_scratch_t *scratch, match_event_handler onEvent, - void *context); - -/** - * The vectored regular expression scanner. - * - * This is the function call in which the actual pattern matching takes place - * for vectoring-mode pattern databases. - * - * @param db - * A compiled pattern database. - * - * @param data - * An array of pointers to the data blocks to be scanned. - * - * @param length - * An array of lengths (in bytes) of each data block to scan. - * - * @param count - * Number of data blocks to scan. This should correspond to the size of - * of the @p data and @p length arrays. - * - * @param flags - * Flags modifying the behaviour of this function. This parameter is - * provided for future use and is unused at present. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch() for - * this database. - * - * @param onEvent - * Pointer to a match event callback function. If a NULL pointer is given, - * no matches will be returned. - * - * @param context - * The user defined pointer which will be passed to the callback function. - * - * @return - * Returns @ref HS_SUCCESS on success; @ref HS_SCAN_TERMINATED if the match - * callback indicated that scanning should stop; other values on error. - */ -hs_error_t HS_CDECL hs_scan_vector(const hs_database_t *db, - const char *const *data, - const unsigned int *length, - unsigned int count, unsigned int flags, - hs_scratch_t *scratch, - match_event_handler onEvent, void *context); - -/** - * Allocate a "scratch" space for use by Hyperscan. - * - * This is required for runtime use, and one scratch space per thread, or - * concurrent caller, is required. Any allocator callback set by @ref - * hs_set_scratch_allocator() or @ref hs_set_allocator() will be used by this - * function. - * - * @param db - * The database, as produced by @ref hs_compile(). - * - * @param scratch - * On first allocation, a pointer to NULL should be provided so a new - * scratch can be allocated. If a scratch block has been previously - * allocated, then a pointer to it should be passed back in to see if it - * is valid for this database block. If a new scratch block is required, - * the original will be freed and the new one returned, otherwise the - * previous scratch block will be returned. On success, the scratch block - * will be suitable for use with the provided database in addition to any - * databases that original scratch space was suitable for. - * - * @return - * @ref HS_SUCCESS on successful allocation; @ref HS_NOMEM if the - * allocation fails. Other errors may be returned if invalid parameters - * are specified. - */ -hs_error_t HS_CDECL hs_alloc_scratch(const hs_database_t *db, - hs_scratch_t **scratch); - -/** - * Allocate a scratch space that is a clone of an existing scratch space. - * - * This is useful when multiple concurrent threads will be using the same set - * of compiled databases, and another scratch space is required. Any allocator - * callback set by @ref hs_set_scratch_allocator() or @ref hs_set_allocator() - * will be used by this function. - * - * @param src - * The existing @ref hs_scratch_t to be cloned. - * - * @param dest - * A pointer to the new scratch space will be returned here. - * - * @return - * @ref HS_SUCCESS on success; @ref HS_NOMEM if the allocation fails. - * Other errors may be returned if invalid parameters are specified. - */ -hs_error_t HS_CDECL hs_clone_scratch(const hs_scratch_t *src, - hs_scratch_t **dest); - -/** - * Provides the size of the given scratch space. - * - * @param scratch - * A per-thread scratch space allocated by @ref hs_alloc_scratch() or @ref - * hs_clone_scratch(). - * - * @param scratch_size - * On success, the size of the scratch space in bytes is placed in this - * parameter. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_scratch_size(const hs_scratch_t *scratch, - size_t *scratch_size); - -/** - * Free a scratch block previously allocated by @ref hs_alloc_scratch() or @ref - * hs_clone_scratch(). - * - * The free callback set by @ref hs_set_scratch_allocator() or @ref - * hs_set_allocator() will be used by this function. - * - * @param scratch - * The scratch block to be freed. NULL may also be safely provided. - * - * @return - * @ref HS_SUCCESS on success, other values on failure. - */ -hs_error_t HS_CDECL hs_free_scratch(hs_scratch_t *scratch); - -/** - * Callback 'from' return value, indicating that the start of this match was - * too early to be tracked with the requested SOM_HORIZON precision. - */ -#define HS_OFFSET_PAST_HORIZON (~0ULL) - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* HS_RUNTIME_H_ */ diff --git a/src/inc_internal/view_only/rulescan.h b/src/inc_internal/view_only/rulescan.h deleted file mode 100644 index f377cc4..0000000 --- a/src/inc_internal/view_only/rulescan.h +++ /dev/null @@ -1,322 +0,0 @@ -/* - * - * Copyright (c) 2014 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LIU YANBING (liuyanbing@iie.ac.cn) - * Last modification: 2016-06-05 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef H_RULE_SCAN_H -#define H_RULE_SCAN_H - -#ifdef __cplusplus -extern "C" -{ -#endif - - /* rulescan_set_param函数å¯è®¾ç½®çš„å‚数类型 */ - enum RULESCAN_PARA_NAME - { - RULESCAN_DETAIL_RESULT=1, /* 本标志ä½è¡¨ç¤ºï¼šè¿”回详细命中ä½ç½®ç­‰ä¿¡æ¯, optval设为NULL，optlen设为0。默认ä¸è¿”回详细信æ¯*/ - RULESCAN_REGEX_GROUP =2, /* 本标志ä½è¡¨ç¤ºï¼šè¿”回正则表达å¼åŒ¹é…的分组信æ¯ï¼›å¼€å¯æœ¬å­—段，需è¦å…ˆè®¾ç½®RULESCAN_DETAIL_RESULT标志ä½,optval设为NULL，optlen设为0。默认ä¸è¿”å›žåˆ†ç»„ä¿¡æ¯ */ - RULEACAN_ERRLOG_CLOSE, /* 本标志ä½è¡¨ç¤ºï¼šå…³é—­Rulescan错误日志输出，optval设为NULL，optlen设为0。ä¸è®¾ç½®çš„è¯é»˜è®¤æ‰“å¼€Rulescan错误日志输出 */ - RULESCAN_ERRLOG_FILE_PATH, /* 设置Rulescan错误日志的路径å（包å«æ–‡ä»¶å），由用户传入，optval的值为包å«æ–‡ä»¶å的日志路径，optlen为路径长度。如果没有设定， - 则日志默认存储在å¯æ‰§è¡Œç¨‹åºå½“å‰ç›®å½•ä¸‹çš„rulescan_tmp中 */ - }; - - #define MAX_REGEX_GROUP_NUM 5 /* 对于正则表达å¼ï¼Œæ‰€æ”¯æŒçš„最大分组的个数 */ - - #define MAX_EXPR_ITEM_NUM (1U<<3) /* æ¯æ¡ä¸Žè¡¨è¾¾å¼æœ€å¤šç”±MAX_EXPR_ITEM_NUMä¸ªè§„åˆ™ç»„æˆ */ - #define MAX_MATCH_POS_NUM 1024 /* æ¯æ¡è§„则最多å…许返回的命中ä½ç½®çš„个数 */ - #define MATCH_POS_NUM_INC 64 /* æ¯æ¡è§„则å…许返回的命中ä½ç½®çš„个数åˆå§‹å€¼ä¸Žå¢žé‡å€¼ */ - - /* 定义ä¸åŒçš„规则类型 */ - const unsigned int RULETYPE_STR = 0; /* 字符串或二进制规则 */ - const unsigned int RULETYPE_REG = 1; /* 正则表达å¼è§„则 */ - const unsigned int RULETYPE_INT = 2; /* 数值区间规则 */ - const unsigned int RULETYPE_IPv4 = 3; /* IPv4规则 */ - const unsigned int RULETYPE_IPv6 = 4; /* IPv6规则 */ - - const unsigned int MAX_RULETYPE = 5; /* è§„åˆ™ç±»åž‹æ•°é‡ */ - const unsigned int MAX_SUB_RULETYPE = 4096; /* 规则å­ç±»åž‹æ•°é‡ */ - - /* 字符串类型规则（å¯è¡¨ç¤ºæ–‡æœ¬å­—符串ã€äºŒè¿›åˆ¶å­—符串ã€æ­£åˆ™è¡¨è¾¾å¼ï¼‰ */ - typedef struct _string_rule_t - { - char * str; /* 字符串内容；如果是正则表达å¼ï¼Œé¡»ä»¥'\0'结æŸï¼Œå¯ä¸æŒ‡å®šé•¿åº¦ */ - unsigned int len; /* 字符串长度 */ - unsigned char case_sensitive; /* 是å¦å¤§å°å†™æ•æ„ŸåŒ¹é…（1：æ•æ„Ÿï¼›0：ä¸æ•æ„Ÿï¼‰ */ - unsigned char match_mode; /* 匹é…模å¼ï¼šå­ä¸²åŒ¹é…（0），完整匹é…（1）；仅对精确串匹é…有效 */ - int l_offset; /* 表示模å¼ä¸²åªèƒ½åœ¨æ–‡æœ¬èŒƒå›´[l_offset, r_offset]中出现，-1表示无é™åˆ¶,-2表示左匹é…；仅对精确串匹é…有效 */ - int r_offset; /* 表示模å¼ä¸²åªèƒ½åœ¨æ–‡æœ¬èŒƒå›´[l_offset, r_offset]中出现，-1表示无é™åˆ¶,-2表示å³åŒ¹é…；仅对精确串匹é…有效 */ - }string_rule_t; - - /* 整数数值区间规则，表示整数区间[lb, ub] */ - typedef struct _interval_rule_t - { - unsigned int lb; /* æ•°æ®åŒºé—´çš„下界（包å«lb），无é™åˆ¶é»˜è®¤ä¸º0 */ - unsigned int ub; /* æ•°æ®åŒºé—´çš„下界（包å«ub），无é™åˆ¶é»˜è®¤ä¸º0 */ - }interval_rule_t; - - /* IPv4规则 */ - typedef struct _ipv4_rule_t - { - unsigned int min_saddr; /* æºåœ°å€ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned int max_saddr; /* æºåœ°å€ä¸Šç•Œï¼›0表示固定IP=min_saddr */ - unsigned int min_daddr; /* 目的地å€ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned int max_daddr; /* 目的地å€ä¸Šç•Œï¼›0表示固定IP=min_daddr */ - unsigned short min_sport; /* æºç«¯å£èŒƒå›´ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned short max_sport; /* æºç«¯å£èŒƒå›´ä¸Šç•Œï¼›0表示固定端å£=min_sport */ - unsigned short min_dport; /* 目的端å£èŒƒå›´ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned short max_dport; /* 目的端å£èŒƒå›´ä¸Šç•Œï¼›0表示固定端å£=min_dport */ - unsigned short proto; /* 传输层å议，6表示TCP，17表示UDPï¼›0表示忽略本字段 */ - unsigned short direction; /* æ–¹å‘，0表示åŒå‘，1表示å•å‘ */ - }ipv4_rule_t; - - /* IPv6规则 */ - typedef struct _ipv6_rule_t - { - unsigned int min_saddr[4]; /* æºåœ°å€ä¸‹ç•Œï¼›å…¨0表示忽略本字段 */ - unsigned int max_saddr[4]; /* æºåœ°å€ä¸Šç•Œï¼›å…¨0表示固定IP=min_saddr */ - unsigned int min_daddr[4]; /* 目的地å€ä¸‹ç•Œï¼›å…¨0表示忽略本字段 */ - unsigned int max_daddr[4]; /* 目的地å€ä¸Šç•Œï¼›å…¨0表示固定IP=min_daddr */ - unsigned short min_sport; /* æºç«¯å£èŒƒå›´ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned short max_sport; /* æºç«¯å£èŒƒå›´ä¸Šç•Œï¼›0表示固定端å£=min_sport */ - unsigned short min_dport; /* 目的端å£èŒƒå›´ä¸‹ç•Œï¼›0表示忽略本字段 */ - unsigned short max_dport; /* 目的端å£èŒƒå›´ä¸Šç•Œï¼›0表示固定端å£=min_dport */ - unsigned short proto; /* 传输层å议，6表示TCP，17表示UDP，无é™åˆ¶é»˜è®¤ä¸º0 */ - unsigned short direction; /* æ–¹å‘，0表示åŒå‘，1表示å•å‘ */ - }ipv6_rule_t; - - /* 通用的规则类型 */ - typedef struct _scan_rule_t - { - unsigned int rule_type; /* 规则类型，必须为上述枚举规则类型之一 */ - unsigned int sub_type; /* å­ç±»ç±»åž‹ï¼Œç”¨æˆ·è‡ªå®šä¹‰ï¼Œä½†å­ç±»åž‹ä¸ªæ•°ä¸å…许超过MAX_SUB_RULETYPE（è§å‰æ–‡å®šä¹‰ï¼‰ */ - union /* æ ¹æ®rule_type决定规则是字符串ã€æ•°å€¼åŒºé—´ã€è¿˜æ˜¯IP规则 */ - { - string_rule_t string_rule; /* 字符串规则（字符串ã€äºŒè¿›åˆ¶ã€æ­£åˆ™è¡¨è¾¾å¼ï¼‰ */ - interval_rule_t interval_rule; /* 整数数值区间规则 */ - ipv4_rule_t ipv4_rule; /* 带掩ç çš„IPv4规则 */ - ipv6_rule_t ipv6_rule; /* 带掩ç çš„IPv6规则 */ - }; - }scan_rule_t; - - /* 一æ¡ä¸Žè¡¨è¾¾å¼è§„则 */ - typedef struct _boolean_expr_t - { - unsigned int expr_id; /* 与表达å¼çš„ID */ - unsigned int operation; /* 对与表达å¼æ‰§è¡Œçš„æ“作：0表示增加，1表示删除 */ - unsigned int rnum; /* 该与表达å¼åŒ…å«å¤šå°‘个项；如果operation=1，置rnum=0å³å¯ */ - scan_rule_t * rules; /* 组æˆä¸Žè¡¨è¾¾å¼çš„项；如果operation=1，置rules=NULLå³å¯ */ - void * tag; /* 用户自定义数æ®ï¼Œå‘½ä¸­æ—¶éšåŒ¹é…结果返回 */ - }boolean_expr_t; - - - /* 待扫æ的文本数æ®ç±»åž‹ */ - typedef struct _text_data_t - { - const char * text; /* 文本数æ®å†…容 */ - unsigned int tlen; /* 文本数æ®é•¿åº¦ */ - int toffset;/* 本段文本数æ®åœ¨æ•´ä¸ªæµæ•°æ®ä¸­çš„å移é‡ï¼Œæµå¼æ‰«æ情况下有效，由用户传入，其它情况置为0(这个必须置为0) */ - }text_data_t; - - /* 待扫æçš„IPv4元组 */ - typedef struct _ipv4_data_t - { - unsigned int saddr; /* æºIPåœ°å€ */ - unsigned int daddr; /* 目的IPåœ°å€ */ - unsigned short int sport; /* æºç«¯å£ */ - unsigned short int dport; /* ç›®çš„ç«¯å£ */ - unsigned short int proto; /* 传输层å议，6表示TCP，17表示UDP */ - }ipv4_data_t; - - /* 待扫æçš„IPv6元组 */ - typedef struct _ipv6_data_t - { - unsigned int saddr[4]; /* æºIPåœ°å€ */ - unsigned int daddr[4]; /* 目的IPåœ°å€ */ - unsigned short int sport; /* æºç«¯å£ */ - unsigned short int dport; /* ç›®çš„ç«¯å£ */ - unsigned short int proto; /* 传输层å议，6表示TCP，17表示UDP */ - }ipv6_data_t; - - /* 通用的待扫ææ•°æ®ç±»åž‹ */ - typedef struct _scan_data_t - { - unsigned int rule_type; /* 规则类型，必须为上述枚举规则类型之一 */ - unsigned int sub_type; /* å­ç±»ç±»åž‹ï¼Œç”¨æˆ·è‡ªå®šä¹‰ï¼Œä½†å­ç±»åž‹ä¸ªæ•°ä¸å…许超过MAX_SUB_RULETYPE（è§å‰æ–‡å®šä¹‰ï¼‰ */ - union /* æ ¹æ®rule_type决定数æ®è´Ÿè½½æ˜¯å­—符串ã€æ•°å€¼ã€è¿˜æ˜¯IP元组 */ - { - text_data_t text_data; /* 待扫æ文本数æ®ï¼ˆå¯åŒ¹é…字符串ã€äºŒè¿›åˆ¶ã€æ­£åˆ™è¡¨è¾¾å¼ï¼‰ */ - unsigned int int_data; /* 整数数值（å¯åŒ¹é…数值区间） */ - ipv4_data_t ipv4_data; /* 待扫æçš„IPv4元组 */ - ipv6_data_t ipv6_data; /* 待扫æçš„IPv6元组 */ - }; - }scan_data_t; - - /* - 扫æ结果类型scan_result_tå’Œrule_result_t说明： - 1ã€å¯¹äºŽå‘½ä¸­çš„æ¯ä¸ªå¸ƒå°”表达å¼ï¼Œä¸€æ¡è¡¨è¾¾å¼å¯¹åº”一æ¡scan_result_t结果，该布尔表达å¼åŒ…å«rnum个规则，æ¯ä¸ªè§„则å‡å¯¹åº”于一个结果scan_result_t::result[k]（0<=k -#include - -struct zt_state_t -{ - union - { - unsigned char matrix[4]; //strong hash state - unsigned int val; - }; -}; - -const unsigned char zt_multi_table[256][4] = -{ - {76,28,128,81},{76,204,128,209},{204,128,209,81},{204,76,209,128},{238,209,196,115},{238,63,196,183},{63,209,183,115},{63,238,183,196},{230,196,193,123},{230,34,193,186},{34,196,186,123},{34,230,186,193},{0,183,175,76},{0,183,175,227},{183,183,227,76},{183,0,227,175},{228,193,192,121},{228,37,192,185},{37,193,185,121},{37,228,185,192},{18,186,164,75},{18,168,164,239},{168,186,239,75},{168,18,239,164},{15,175,169,67},{15,160,169,234},{160,175,234,67},{160,15,234,169},{151,227,247,108},{151,116,247,155},{116,227,155,108},{116,151,155,247},{228,192,193,121},{228,36,193,184},{36,192,184,121},{36,228,184,193},{22,185,167,74},{22,175,167,237},{175,185,237,74},{175,22,237,167},{30,164,162,71},{30,186,162,229},{186,164,229,71},{186,30,229,162},{136,239,250,107},{136,103,250,145},{103,239,145,107},{103,136,145,250},{12,169,169,64},{12,165,169,233},{165,169,233,64},{165,12,233,169},{138,234,251,105},{138,96,251,146},{96,234,146,105},{96,138,146,251},{159,247,243,100},{159,104,243,151},{104,247,151,100},{104,159,151,243},{71,155,133,95},{71,220,133,218},{220,155,218,95},{220,71,218,133},{230,193,196,123},{230,39,196,191},{39,193,191,123},{39,230,191,196},{20,184,160,73},{20,172,160,233},{172,184,233,73},{172,20,233,160},{25,167,167,69},{25,190,167,226},{190,167,226,69},{190,25,226,167},{141,237,253,104},{141,96,253,149},{96,237,149,104},{96,141,149,253},{30,162,164,71},{30,188,164,227},{188,162,227,71},{188,30,227,164},{144,229,240,109},{144,117,240,157},{117,229,157,109},{117,144,157,240},{130,250,251,97},{130,120,251,154},{120,250,154,97},{120,130,154,251},{84,145,137,88},{84,197,137,209},{197,145,209,88},{197,84,209,137},{15,169,175,67},{15,166,175,236},{166,169,236,67},{166,15,236,175},{143,233,253,106},{143,102,253,151},{102,233,151,106},{102,143,151,253},{130,251,250,97},{130,121,250,155},{121,251,155,97},{121,130,155,250},{80,146,138,89},{80,194,138,211},{194,146,211,89},{194,80,211,138},{159,243,247,100},{159,108,247,147},{108,243,147,100},{108,159,147,247},{87,151,137,91},{87,192,137,210},{192,151,210,91},{192,87,210,137},{72,133,133,80},{72,205,133,213},{205,133,213,80},{205,72,213,133},{246,218,207,117},{246,44,207,186},{44,218,186,117},{44,246,186,207},{238,196,209,115},{238,42,209,162},{42,196,162,115},{42,238,162,209},{24,191,191,68},{24,167,191,251},{167,191,251,68},{167,24,251,191},{20,160,184,73},{20,180,184,241},{180,160,241,73},{180,20,241,184},{134,233,236,99},{134,111,236,143},{111,233,143,99},{111,134,143,236},{22,167,185,74},{22,177,185,243},{177,167,243,74},{177,22,243,185},{156,226,227,103},{156,126,227,132},{126,226,132,103},{126,156,132,227},{143,253,233,106},{143,114,233,131},{114,253,131,106},{114,143,131,233},{95,149,147,87},{95,202,147,196},{202,149,196,87},{202,95,196,147},{18,164,186,75},{18,182,186,241},{182,164,241,75},{182,18,241,186},{156,227,226,103},{156,127,226,133},{127,227,133,103},{127,156,133,226},{144,240,229,109},{144,96,229,136},{96,240,136,109},{96,144,136,229},{74,157,155,82},{74,215,155,201},{215,157,201,82},{215,74,201,155},{138,251,234,105},{138,113,234,131},{113,251,131,105},{113,138,131,234},{72,154,154,81},{72,210,154,203},{210,154,203,81},{210,72,203,154},{87,137,151,91},{87,222,151,204},{222,137,204,91},{222,87,204,151},{231,209,213,122},{231,54,213,175},{54,209,175,122},{54,231,175,213},{0,175,183,76},{0,175,183,251},{175,175,251,76},{175,0,251,183},{134,236,233,99},{134,106,233,138},{106,236,138,99},{106,134,138,233},{141,253,237,104},{141,112,237,133},{112,253,133,104},{112,141,133,237},{89,151,151,85},{89,206,151,194},{206,151,194,85},{206,89,194,151},{136,250,239,107},{136,114,239,132},{114,250,132,107},{114,136,132,239},{74,155,157,82},{74,209,157,207},{209,155,207,82},{209,74,207,157},{80,138,146,89},{80,218,146,203},{218,138,203,89},{218,80,203,146},{226,211,210,121},{226,49,210,171},{49,211,171,121},{49,226,171,210},{151,247,227,108},{151,96,227,143},{96,247,143,108},{96,151,143,227},{95,147,149,87},{95,204,149,194},{204,147,194,87},{204,95,194,149},{84,137,145,88},{84,221,145,201},{221,137,201,88},{221,84,201,145},{226,210,211,121},{226,48,211,170},{48,210,170,121},{48,226,170,211},{71,133,155,95},{71,194,155,196},{194,133,196,95},{194,71,196,155},{231,213,209,122},{231,50,209,171},{50,213,171,122},{50,231,171,209},{246,207,218,117},{246,57,218,175},{57,207,175,117},{57,246,175,218},{28,186,186,69},{28,166,186,255},{166,186,255,69},{166,28,255,186} -}; - -const static unsigned char galois_mult_8_table[65536] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128,130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164,166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200,202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236,238,240,242,244,246,248,250,252,254,29,31,25,27,21,23,17,19,13,15,9,11,5,7,1,3,61,63,57,59,53,55,49,51,45,47,41,43,37,39,33,35,93,95,89,91,85,87,81,83,77,79,73,75,69,71,65,67,125,127,121,123,117,119,113,115,109,111,105,107,101,103,97,99,157,159,153,155,149,151,145,147,141,143,137,139,133,135,129,131,189,191,185,187,181,183,177,179,173,175,169,171,165,167,161,163,221,223,217,219,213,215,209,211,205,207,201,203,197,199,193,195,253,255,249,251,245,247,241,243,237,239,233,235,229,231,225,227,0,3,6,5,12,15,10,9,24,27,30,29,20,23,18,17,48,51,54,53,60,63,58,57,40,43,46,45,36,39,34,33,96,99,102,101,108,111,106,105,120,123,126,125,116,119,114,113,80,83,86,85,92,95,90,89,72,75,78,77,68,71,66,65,192,195,198,197,204,207,202,201,216,219,222,221,212,215,210,209,240,243,246,245,252,255,250,249,232,235,238,237,228,231,226,225,160,163,166,165,172,175,170,169,184,187,190,189,180,183,178,177,144,147,150,149,156,159,154,153,136,139,142,141,132,135,130,129,157,158,155,152,145,146,151,148,133,134,131,128,137,138,143,140,173,174,171,168,161,162,167,164,181,182,179,176,185,186,191,188,253,254,251,248,241,242,247,244,229,230,227,224,233,234,239,236,205,206,203,200,193,194,199,196,213,214,211,208,217,218,223,220,93,94,91,88,81,82,87,84,69,70,67,64,73,74,79,76,109,110,107,104,97,98,103,100,117,118,115,112,121,122,127,124,61,62,59,56,49,50,55,52,37,38,35,32,41,42,47,44,13,14,11,8,1,2,7,4,21,22,19,16,25,26,31,28,0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,132,136,140,144,148,152,156,160,164,168,172,176,180,184,188,192,196,200,204,208,212,216,220,224,228,232,236,240,244,248,252,29,25,21,17,13,9,5,1,61,57,53,49,45,41,37,33,93,89,85,81,77,73,69,65,125,121,117,113,109,105,101,97,157,153,149,145,141,137,133,129,189,185,181,177,173,169,165,161,221,217,213,209,205,201,197,193,253,249,245,241,237,233,229,225,58,62,50,54,42,46,34,38,26,30,18,22,10,14,2,6,122,126,114,118,106,110,98,102,90,94,82,86,74,78,66,70,186,190,178,182,170,174,162,166,154,158,146,150,138,142,130,134,250,254,242,246,234,238,226,230,218,222,210,214,202,206,194,198,39,35,47,43,55,51,63,59,7,3,15,11,23,19,31,27,103,99,111,107,119,115,127,123,71,67,79,75,87,83,95,91,167,163,175,171,183,179,191,187,135,131,143,139,151,147,159,155,231,227,239,235,247,243,255,251,199,195,207,203,215,211,223,219,0,5,10,15,20,17,30,27,40,45,34,39,60,57,54,51,80,85,90,95,68,65,78,75,120,125,114,119,108,105,102,99,160,165,170,175,180,177,190,187,136,141,130,135,156,153,150,147,240,245,250,255,228,225,238,235,216,221,210,215,204,201,198,195,93,88,87,82,73,76,67,70,117,112,127,122,97,100,107,110,13,8,7,2,25,28,19,22,37,32,47,42,49,52,59,62,253,248,247,242,233,236,227,230,213,208,223,218,193,196,203,206,173,168,167,162,185,188,179,182,133,128,143,138,145,148,155,158,186,191,176,181,174,171,164,161,146,151,152,157,134,131,140,137,234,239,224,229,254,251,244,241,194,199,200,205,214,211,220,217,26,31,16,21,14,11,4,1,50,55,56,61,38,35,44,41,74,79,64,69,94,91,84,81,98,103,104,109,118,115,124,121,231,226,237,232,243,246,249,252,207,202,197,192,219,222,209,212,183,178,189,184,163,166,169,172,159,154,149,144,139,142,129,132,71,66,77,72,83,86,89,92,111,106,101,96,123,126,113,116,23,18,29,24,3,6,9,12,63,58,53,48,43,46,33,36,0,6,12,10,24,30,20,18,48,54,60,58,40,46,36,34,96,102,108,106,120,126,116,114,80,86,92,90,72,78,68,66,192,198,204,202,216,222,212,210,240,246,252,250,232,238,228,226,160,166,172,170,184,190,180,178,144,150,156,154,136,142,132,130,157,155,145,151,133,131,137,143,173,171,161,167,181,179,185,191,253,251,241,247,229,227,233,239,205,203,193,199,213,211,217,223,93,91,81,87,69,67,73,79,109,107,97,103,117,115,121,127,61,59,49,55,37,35,41,47,13,11,1,7,21,19,25,31,39,33,43,45,63,57,51,53,23,17,27,29,15,9,3,5,71,65,75,77,95,89,83,85,119,113,123,125,111,105,99,101,231,225,235,237,255,249,243,245,215,209,219,221,207,201,195,197,135,129,139,141,159,153,147,149,183,177,187,189,175,169,163,165,186,188,182,176,162,164,174,168,138,140,134,128,146,148,158,152,218,220,214,208,194,196,206,200,234,236,230,224,242,244,254,248,122,124,118,112,98,100,110,104,74,76,70,64,82,84,94,88,26,28,22,16,2,4,14,8,42,44,38,32,50,52,62,56,0,7,14,9,28,27,18,21,56,63,54,49,36,35,42,45,112,119,126,121,108,107,98,101,72,79,70,65,84,83,90,93,224,231,238,233,252,251,242,245,216,223,214,209,196,195,202,205,144,151,158,153,140,139,130,133,168,175,166,161,180,179,186,189,221,218,211,212,193,198,207,200,229,226,235,236,249,254,247,240,173,170,163,164,177,182,191,184,149,146,155,156,137,142,135,128,61,58,51,52,33,38,47,40,5,2,11,12,25,30,23,16,77,74,67,68,81,86,95,88,117,114,123,124,105,110,103,96,167,160,169,174,187,188,181,178,159,152,145,150,131,132,141,138,215,208,217,222,203,204,197,194,239,232,225,230,243,244,253,250,71,64,73,78,91,92,85,82,127,120,113,118,99,100,109,106,55,48,57,62,43,44,37,34,15,8,1,6,19,20,29,26,122,125,116,115,102,97,104,111,66,69,76,75,94,89,80,87,10,13,4,3,22,17,24,31,50,53,60,59,46,41,32,39,154,157,148,147,134,129,136,143,162,165,172,171,190,185,176,183,234,237,228,227,246,241,248,255,210,213,220,219,206,201,192,199,0,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,29,21,13,5,61,53,45,37,93,85,77,69,125,117,109,101,157,149,141,133,189,181,173,165,221,213,205,197,253,245,237,229,58,50,42,34,26,18,10,2,122,114,106,98,90,82,74,66,186,178,170,162,154,146,138,130,250,242,234,226,218,210,202,194,39,47,55,63,7,15,23,31,103,111,119,127,71,79,87,95,167,175,183,191,135,143,151,159,231,239,247,255,199,207,215,223,116,124,100,108,84,92,68,76,52,60,36,44,20,28,4,12,244,252,228,236,212,220,196,204,180,188,164,172,148,156,132,140,105,97,121,113,73,65,89,81,41,33,57,49,9,1,25,17,233,225,249,241,201,193,217,209,169,161,185,177,137,129,153,145,78,70,94,86,110,102,126,118,14,6,30,22,46,38,62,54,206,198,222,214,238,230,254,246,142,134,158,150,174,166,190,182,83,91,67,75,115,123,99,107,19,27,3,11,51,59,35,43,211,219,195,203,243,251,227,235,147,155,131,139,179,187,163,171,0,9,18,27,36,45,54,63,72,65,90,83,108,101,126,119,144,153,130,139,180,189,166,175,216,209,202,195,252,245,238,231,61,52,47,38,25,16,11,2,117,124,103,110,81,88,67,74,173,164,191,182,137,128,155,146,229,236,247,254,193,200,211,218,122,115,104,97,94,87,76,69,50,59,32,41,22,31,4,13,234,227,248,241,206,199,220,213,162,171,176,185,134,143,148,157,71,78,85,92,99,106,113,120,15,6,29,20,43,34,57,48,215,222,197,204,243,250,225,232,159,150,141,132,187,178,169,160,244,253,230,239,208,217,194,203,188,181,174,167,152,145,138,131,100,109,118,127,64,73,82,91,44,37,62,55,8,1,26,19,201,192,219,210,237,228,255,246,129,136,147,154,165,172,183,190,89,80,75,66,125,116,111,102,17,24,3,10,53,60,39,46,142,135,156,149,170,163,184,177,198,207,212,221,226,235,240,249,30,23,12,5,58,51,40,33,86,95,68,77,114,123,96,105,179,186,161,168,151,158,133,140,251,242,233,224,223,214,205,196,35,42,49,56,7,14,21,28,107,98,121,112,79,70,93,84,0,10,20,30,40,34,60,54,80,90,68,78,120,114,108,102,160,170,180,190,136,130,156,150,240,250,228,238,216,210,204,198,93,87,73,67,117,127,97,107,13,7,25,19,37,47,49,59,253,247,233,227,213,223,193,203,173,167,185,179,133,143,145,155,186,176,174,164,146,152,134,140,234,224,254,244,194,200,214,220,26,16,14,4,50,56,38,44,74,64,94,84,98,104,118,124,231,237,243,249,207,197,219,209,183,189,163,169,159,149,139,129,71,77,83,89,111,101,123,113,23,29,3,9,63,53,43,33,105,99,125,119,65,75,85,95,57,51,45,39,17,27,5,15,201,195,221,215,225,235,245,255,153,147,141,135,177,187,165,175,52,62,32,42,28,22,8,2,100,110,112,122,76,70,88,82,148,158,128,138,188,182,168,162,196,206,208,218,236,230,248,242,211,217,199,205,251,241,239,229,131,137,151,157,171,161,191,181,115,121,103,109,91,81,79,69,35,41,55,61,11,1,31,21,142,132,154,144,166,172,178,184,222,212,202,192,246,252,226,232,46,36,58,48,6,12,18,24,126,116,106,96,86,92,66,72,0,11,22,29,44,39,58,49,88,83,78,69,116,127,98,105,176,187,166,173,156,151,138,129,232,227,254,245,196,207,210,217,125,118,107,96,81,90,71,76,37,46,51,56,9,2,31,20,205,198,219,208,225,234,247,252,149,158,131,136,185,178,175,164,250,241,236,231,214,221,192,203,162,169,180,191,142,133,152,147,74,65,92,87,102,109,112,123,18,25,4,15,62,53,40,35,135,140,145,154,171,160,189,182,223,212,201,194,243,248,229,238,55,60,33,42,27,16,13,6,111,100,121,114,67,72,85,94,233,226,255,244,197,206,211,216,177,186,167,172,157,150,139,128,89,82,79,68,117,126,99,104,1,10,23,28,45,38,59,48,148,159,130,137,184,179,174,165,204,199,218,209,224,235,246,253,36,47,50,57,8,3,30,21,124,119,106,97,80,91,70,77,19,24,5,14,63,52,41,34,75,64,93,86,103,108,113,122,163,168,181,190,143,132,153,146,251,240,237,230,215,220,193,202,110,101,120,115,66,73,84,95,54,61,32,43,26,17,12,7,222,213,200,195,242,249,228,239,134,141,144,155,170,161,188,183,0,12,24,20,48,60,40,36,96,108,120,116,80,92,72,68,192,204,216,212,240,252,232,228,160,172,184,180,144,156,136,132,157,145,133,137,173,161,181,185,253,241,229,233,205,193,213,217,93,81,69,73,109,97,117,121,61,49,37,41,13,1,21,25,39,43,63,51,23,27,15,3,71,75,95,83,119,123,111,99,231,235,255,243,215,219,207,195,135,139,159,147,183,187,175,163,186,182,162,174,138,134,146,158,218,214,194,206,234,230,242,254,122,118,98,110,74,70,82,94,26,22,2,14,42,38,50,62,78,66,86,90,126,114,102,106,46,34,54,58,30,18,6,10,142,130,150,154,190,178,166,170,238,226,246,250,222,210,198,202,211,223,203,199,227,239,251,247,179,191,171,167,131,143,155,151,19,31,11,7,35,47,59,55,115,127,107,103,67,79,91,87,105,101,113,125,89,85,65,77,9,5,17,29,57,53,33,45,169,165,177,189,153,149,129,141,201,197,209,221,249,245,225,237,244,248,236,224,196,200,220,208,148,152,140,128,164,168,188,176,52,56,44,32,4,8,28,16,84,88,76,64,100,104,124,112,0,13,26,23,52,57,46,35,104,101,114,127,92,81,70,75,208,221,202,199,228,233,254,243,184,181,162,175,140,129,150,155,189,176,167,170,137,132,147,158,213,216,207,194,225,236,251,246,109,96,119,122,89,84,67,78,5,8,31,18,49,60,43,38,103,106,125,112,83,94,73,68,15,2,21,24,59,54,33,44,183,186,173,160,131,142,153,148,223,210,197,200,235,230,241,252,218,215,192,205,238,227,244,249,178,191,168,165,134,139,156,145,10,7,16,29,62,51,36,41,98,111,120,117,86,91,76,65,206,195,212,217,250,247,224,237,166,171,188,177,146,159,136,133,30,19,4,9,42,39,48,61,118,123,108,97,66,79,88,85,115,126,105,100,71,74,93,80,27,22,1,12,47,34,53,56,163,174,185,180,151,154,141,128,203,198,209,220,255,242,229,232,169,164,179,190,157,144,135,138,193,204,219,214,245,248,239,226,121,116,99,110,77,64,87,90,17,28,11,6,37,40,63,50,20,25,14,3,32,45,58,55,124,113,102,107,72,69,82,95,196,201,222,211,240,253,234,231,172,161,182,187,152,149,130,143,0,14,28,18,56,54,36,42,112,126,108,98,72,70,84,90,224,238,252,242,216,214,196,202,144,158,140,130,168,166,180,186,221,211,193,207,229,235,249,247,173,163,177,191,149,155,137,135,61,51,33,47,5,11,25,23,77,67,81,95,117,123,105,103,167,169,187,181,159,145,131,141,215,217,203,197,239,225,243,253,71,73,91,85,127,113,99,109,55,57,43,37,15,1,19,29,122,116,102,104,66,76,94,80,10,4,22,24,50,60,46,32,154,148,134,136,162,172,190,176,234,228,246,248,210,220,206,192,83,93,79,65,107,101,119,121,35,45,63,49,27,21,7,9,179,189,175,161,139,133,151,153,195,205,223,209,251,245,231,233,142,128,146,156,182,184,170,164,254,240,226,236,198,200,218,212,110,96,114,124,86,88,74,68,30,16,2,12,38,40,58,52,244,250,232,230,204,194,208,222,132,138,152,150,188,178,160,174,20,26,8,6,44,34,48,62,100,106,120,118,92,82,64,78,41,39,53,59,17,31,13,3,89,87,69,75,97,111,125,115,201,199,213,219,241,255,237,227,185,183,165,171,129,143,157,147,0,15,30,17,60,51,34,45,120,119,102,105,68,75,90,85,240,255,238,225,204,195,210,221,136,135,150,153,180,187,170,165,253,242,227,236,193,206,223,208,133,138,155,148,185,182,167,168,13,2,19,28,49,62,47,32,117,122,107,100,73,70,87,88,231,232,249,246,219,212,197,202,159,144,129,142,163,172,189,178,23,24,9,6,43,36,53,58,111,96,113,126,83,92,77,66,26,21,4,11,38,41,56,55,98,109,124,115,94,81,64,79,234,229,244,251,214,217,200,199,146,157,140,131,174,161,176,191,211,220,205,194,239,224,241,254,171,164,181,186,151,152,137,134,35,44,61,50,31,16,1,14,91,84,69,74,103,104,121,118,46,33,48,63,18,29,12,3,86,89,72,71,106,101,116,123,222,209,192,207,226,237,252,243,166,169,184,183,154,149,132,139,52,59,42,37,8,7,22,25,76,67,82,93,112,127,110,97,196,203,218,213,248,247,230,233,188,179,162,173,128,143,158,145,201,198,215,216,245,250,235,228,177,190,175,160,141,130,147,156,57,54,39,40,5,10,27,20,65,78,95,80,125,114,99,108,0,16,32,48,64,80,96,112,128,144,160,176,192,208,224,240,29,13,61,45,93,77,125,109,157,141,189,173,221,205,253,237,58,42,26,10,122,106,90,74,186,170,154,138,250,234,218,202,39,55,7,23,103,119,71,87,167,183,135,151,231,247,199,215,116,100,84,68,52,36,20,4,244,228,212,196,180,164,148,132,105,121,73,89,41,57,9,25,233,249,201,217,169,185,137,153,78,94,110,126,14,30,46,62,206,222,238,254,142,158,174,190,83,67,115,99,19,3,51,35,211,195,243,227,147,131,179,163,232,248,200,216,168,184,136,152,104,120,72,88,40,56,8,24,245,229,213,197,181,165,149,133,117,101,85,69,53,37,21,5,210,194,242,226,146,130,178,162,82,66,114,98,18,2,50,34,207,223,239,255,143,159,175,191,79,95,111,127,15,31,47,63,156,140,188,172,220,204,252,236,28,12,60,44,92,76,124,108,129,145,161,177,193,209,225,241,1,17,33,49,65,81,97,113,166,182,134,150,230,246,198,214,38,54,6,22,102,118,70,86,187,171,155,139,251,235,219,203,59,43,27,11,123,107,91,75,0,17,34,51,68,85,102,119,136,153,170,187,204,221,238,255,13,28,47,62,73,88,107,122,133,148,167,182,193,208,227,242,26,11,56,41,94,79,124,109,146,131,176,161,214,199,244,229,23,6,53,36,83,66,113,96,159,142,189,172,219,202,249,232,52,37,22,7,112,97,82,67,188,173,158,143,248,233,218,203,57,40,27,10,125,108,95,78,177,160,147,130,245,228,215,198,46,63,12,29,106,123,72,89,166,183,132,149,226,243,192,209,35,50,1,16,103,118,69,84,171,186,137,152,239,254,205,220,104,121,74,91,44,61,14,31,224,241,194,211,164,181,134,151,101,116,71,86,33,48,3,18,237,252,207,222,169,184,139,154,114,99,80,65,54,39,20,5,250,235,216,201,190,175,156,141,127,110,93,76,59,42,25,8,247,230,213,196,179,162,145,128,92,77,126,111,24,9,58,43,212,197,246,231,144,129,178,163,81,64,115,98,21,4,55,38,217,200,251,234,157,140,191,174,70,87,100,117,2,19,32,49,206,223,236,253,138,155,168,185,75,90,105,120,15,30,45,60,195,210,225,240,135,150,165,180,0,18,36,54,72,90,108,126,144,130,180,166,216,202,252,238,61,47,25,11,117,103,81,67,173,191,137,155,229,247,193,211,122,104,94,76,50,32,22,4,234,248,206,220,162,176,134,148,71,85,99,113,15,29,43,57,215,197,243,225,159,141,187,169,244,230,208,194,188,174,152,138,100,118,64,82,44,62,8,26,201,219,237,255,129,147,165,183,89,75,125,111,17,3,53,39,142,156,170,184,198,212,226,240,30,12,58,40,86,68,114,96,179,161,151,133,251,233,223,205,35,49,7,21,107,121,79,93,245,231,209,195,189,175,153,139,101,119,65,83,45,63,9,27,200,218,236,254,128,146,164,182,88,74,124,110,16,2,52,38,143,157,171,185,199,213,227,241,31,13,59,41,87,69,115,97,178,160,150,132,250,232,222,204,34,48,6,20,106,120,78,92,1,19,37,55,73,91,109,127,145,131,181,167,217,203,253,239,60,46,24,10,116,102,80,66,172,190,136,154,228,246,192,210,123,105,95,77,51,33,23,5,235,249,207,221,163,177,135,149,70,84,98,112,14,28,42,56,214,196,242,224,158,140,186,168,0,19,38,53,76,95,106,121,152,139,190,173,212,199,242,225,45,62,11,24,97,114,71,84,181,166,147,128,249,234,223,204,90,73,124,111,22,5,48,35,194,209,228,247,142,157,168,187,119,100,81,66,59,40,29,14,239,252,201,218,163,176,133,150,180,167,146,129,248,235,222,205,44,63,10,25,96,115,70,85,153,138,191,172,213,198,243,224,1,18,39,52,77,94,107,120,238,253,200,219,162,177,132,151,118,101,80,67,58,41,28,15,195,208,229,246,143,156,169,186,91,72,125,110,23,4,49,34,117,102,83,64,57,42,31,12,237,254,203,216,161,178,135,148,88,75,126,109,20,7,50,33,192,211,230,245,140,159,170,185,47,60,9,26,99,112,69,86,183,164,145,130,251,232,221,206,2,17,36,55,78,93,104,123,154,137,188,175,214,197,240,227,193,210,231,244,141,158,171,184,89,74,127,108,21,6,51,32,236,255,202,217,160,179,134,149,116,103,82,65,56,43,30,13,155,136,189,174,215,196,241,226,3,16,37,54,79,92,105,122,182,165,144,131,250,233,220,207,46,61,8,27,98,113,68,87,0,20,40,60,80,68,120,108,160,180,136,156,240,228,216,204,93,73,117,97,13,25,37,49,253,233,213,193,173,185,133,145,186,174,146,134,234,254,194,214,26,14,50,38,74,94,98,118,231,243,207,219,183,163,159,139,71,83,111,123,23,3,63,43,105,125,65,85,57,45,17,5,201,221,225,245,153,141,177,165,52,32,28,8,100,112,76,88,148,128,188,168,196,208,236,248,211,199,251,239,131,151,171,191,115,103,91,79,35,55,11,31,142,154,166,178,222,202,246,226,46,58,6,18,126,106,86,66,210,198,250,238,130,150,170,190,114,102,90,78,34,54,10,30,143,155,167,179,223,203,247,227,47,59,7,19,127,107,87,67,104,124,64,84,56,44,16,4,200,220,224,244,152,140,176,164,53,33,29,9,101,113,77,89,149,129,189,169,197,209,237,249,187,175,147,135,235,255,195,215,27,15,51,39,75,95,99,119,230,242,206,218,182,162,158,138,70,82,110,122,22,2,62,42,1,21,41,61,81,69,121,109,161,181,137,157,241,229,217,205,92,72,116,96,12,24,36,48,252,232,212,192,172,184,132,144,0,21,42,63,84,65,126,107,168,189,130,151,252,233,214,195,77,88,103,114,25,12,51,38,229,240,207,218,177,164,155,142,154,143,176,165,206,219,228,241,50,39,24,13,102,115,76,89,215,194,253,232,131,150,169,188,127,106,85,64,43,62,1,20,41,60,3,22,125,104,87,66,129,148,171,190,213,192,255,234,100,113,78,91,48,37,26,15,204,217,230,243,152,141,178,167,179,166,153,140,231,242,205,216,27,14,49,36,79,90,101,112,254,235,212,193,170,191,128,149,86,67,124,105,2,23,40,61,82,71,120,109,6,19,44,57,250,239,208,197,174,187,132,145,31,10,53,32,75,94,97,116,183,162,157,136,227,246,201,220,200,221,226,247,156,137,182,163,96,117,74,95,52,33,30,11,133,144,175,186,209,196,251,238,45,56,7,18,121,108,83,70,123,110,81,68,47,58,5,16,211,198,249,236,135,146,173,184,54,35,28,9,98,119,72,93,158,139,180,161,202,223,224,245,225,244,203,222,181,160,159,138,73,92,99,118,29,8,55,34,172,185,134,147,248,237,210,199,4,17,46,59,80,69,122,111,0,22,44,58,88,78,116,98,176,166,156,138,232,254,196,210,125,107,81,71,37,51,9,31,205,219,225,247,149,131,185,175,250,236,214,192,162,180,142,152,74,92,102,112,18,4,62,40,135,145,171,189,223,201,243,229,55,33,27,13,111,121,67,85,233,255,197,211,177,167,157,139,89,79,117,99,1,23,45,59,148,130,184,174,204,218,224,246,36,50,8,30,124,106,80,70,19,5,63,41,75,93,103,113,163,181,143,153,251,237,215,193,110,120,66,84,54,32,26,12,222,200,242,228,134,144,170,188,207,217,227,245,151,129,187,173,127,105,83,69,39,49,11,29,178,164,158,136,234,252,198,208,2,20,46,56,90,76,118,96,53,35,25,15,109,123,65,87,133,147,169,191,221,203,241,231,72,94,100,114,16,6,60,42,248,238,212,194,160,182,140,154,38,48,10,28,126,104,82,68,150,128,186,172,206,216,226,244,91,77,119,97,3,21,47,57,235,253,199,209,179,165,159,137,220,202,240,230,132,146,168,190,108,122,64,86,52,34,24,14,161,183,141,155,249,239,213,195,17,7,61,43,73,95,101,115,0,23,46,57,92,75,114,101,184,175,150,129,228,243,202,221,109,122,67,84,49,38,31,8,213,194,251,236,137,158,167,176,218,205,244,227,134,145,168,191,98,117,76,91,62,41,16,7,183,160,153,142,235,252,197,210,15,24,33,54,83,68,125,106,169,190,135,144,245,226,219,204,17,6,63,40,77,90,99,116,196,211,234,253,152,143,182,161,124,107,82,69,32,55,14,25,115,100,93,74,47,56,1,22,203,220,229,242,151,128,185,174,30,9,48,39,66,85,108,123,166,177,136,159,250,237,212,195,79,88,97,118,19,4,61,42,247,224,217,206,171,188,133,146,34,53,12,27,126,105,80,71,154,141,180,163,198,209,232,255,149,130,187,172,201,222,231,240,45,58,3,20,113,102,95,72,248,239,214,193,164,179,138,157,64,87,110,121,28,11,50,37,230,241,200,223,186,173,148,131,94,73,112,103,2,21,44,59,139,156,165,178,215,192,249,238,51,36,29,10,111,120,65,86,60,43,18,5,96,119,78,89,132,147,170,189,216,207,246,225,81,70,127,104,13,26,35,52,233,254,199,208,181,162,155,140,0,24,48,40,96,120,80,72,192,216,240,232,160,184,144,136,157,133,173,181,253,229,205,213,93,69,109,117,61,37,13,21,39,63,23,15,71,95,119,111,231,255,215,207,135,159,183,175,186,162,138,146,218,194,234,242,122,98,74,82,26,2,42,50,78,86,126,102,46,54,30,6,142,150,190,166,238,246,222,198,211,203,227,251,179,171,131,155,19,11,35,59,115,107,67,91,105,113,89,65,9,17,57,33,169,177,153,129,201,209,249,225,244,236,196,220,148,140,164,188,52,44,4,28,84,76,100,124,156,132,172,180,252,228,204,212,92,68,108,116,60,36,12,20,1,25,49,41,97,121,81,73,193,217,241,233,161,185,145,137,187,163,139,147,219,195,235,243,123,99,75,83,27,3,43,51,38,62,22,14,70,94,118,110,230,254,214,206,134,158,182,174,210,202,226,250,178,170,130,154,18,10,34,58,114,106,66,90,79,87,127,103,47,55,31,7,143,151,191,167,239,247,223,199,245,237,197,221,149,141,165,189,53,45,5,29,85,77,101,125,104,112,88,64,8,16,56,32,168,176,152,128,200,208,248,224,0,25,50,43,100,125,86,79,200,209,250,227,172,181,158,135,141,148,191,166,233,240,219,194,69,92,119,110,33,56,19,10,7,30,53,44,99,122,81,72,207,214,253,228,171,178,153,128,138,147,184,161,238,247,220,197,66,91,112,105,38,63,20,13,14,23,60,37,106,115,88,65,198,223,244,237,162,187,144,137,131,154,177,168,231,254,213,204,75,82,121,96,47,54,29,4,9,16,59,34,109,116,95,70,193,216,243,234,165,188,151,142,132,157,182,175,224,249,210,203,76,85,126,103,40,49,26,3,28,5,46,55,120,97,74,83,212,205,230,255,176,169,130,155,145,136,163,186,245,236,199,222,89,64,107,114,61,36,15,22,27,2,41,48,127,102,77,84,211,202,225,248,183,174,133,156,150,143,164,189,242,235,192,217,94,71,108,117,58,35,8,17,18,11,32,57,118,111,68,93,218,195,232,241,190,167,140,149,159,134,173,180,251,226,201,208,87,78,101,124,51,42,1,24,21,12,39,62,113,104,67,90,221,196,239,246,185,160,139,146,152,129,170,179,252,229,206,215,80,73,98,123,52,45,6,31,0,26,52,46,104,114,92,70,208,202,228,254,184,162,140,150,189,167,137,147,213,207,225,251,109,119,89,67,5,31,49,43,103,125,83,73,15,21,59,33,183,173,131,153,223,197,235,241,218,192,238,244,178,168,134,156,10,16,62,36,98,120,86,76,206,212,250,224,166,188,146,136,30,4,42,48,118,108,66,88,115,105,71,93,27,1,47,53,163,185,151,141,203,209,255,229,169,179,157,135,193,219,245,239,121,99,77,87,17,11,37,63,20,14,32,58,124,102,72,82,196,222,240,234,172,182,152,130,129,155,181,175,233,243,221,199,81,75,101,127,57,35,13,23,60,38,8,18,84,78,96,122,236,246,216,194,132,158,176,170,230,252,210,200,142,148,186,160,54,44,2,24,94,68,106,112,91,65,111,117,51,41,7,29,139,145,191,165,227,249,215,205,79,85,123,97,39,61,19,9,159,133,171,177,247,237,195,217,242,232,198,220,154,128,174,180,34,56,22,12,74,80,126,100,40,50,28,6,64,90,116,110,248,226,204,214,144,138,164,190,149,143,161,187,253,231,201,211,69,95,113,107,45,55,25,3,0,27,54,45,108,119,90,65,216,195,238,245,180,175,130,153,173,182,155,128,193,218,247,236,117,110,67,88,25,2,47,52,71,92,113,106,43,48,29,6,159,132,169,178,243,232,197,222,234,241,220,199,134,157,176,171,50,41,4,31,94,69,104,115,142,149,184,163,226,249,212,207,86,77,96,123,58,33,12,23,35,56,21,14,79,84,121,98,251,224,205,214,151,140,161,186,201,210,255,228,165,190,147,136,17,10,39,60,125,102,75,80,100,127,82,73,8,19,62,37,188,167,138,145,208,203,230,253,1,26,55,44,109,118,91,64,217,194,239,244,181,174,131,152,172,183,154,129,192,219,246,237,116,111,66,89,24,3,46,53,70,93,112,107,42,49,28,7,158,133,168,179,242,233,196,223,235,240,221,198,135,156,177,170,51,40,5,30,95,68,105,114,143,148,185,162,227,248,213,206,87,76,97,122,59,32,13,22,34,57,20,15,78,85,120,99,250,225,204,215,150,141,160,187,200,211,254,229,164,191,146,137,16,11,38,61,124,103,74,81,101,126,83,72,9,18,63,36,189,166,139,144,209,202,231,252,0,28,56,36,112,108,72,84,224,252,216,196,144,140,168,180,221,193,229,249,173,177,149,137,61,33,5,25,77,81,117,105,167,187,159,131,215,203,239,243,71,91,127,99,55,43,15,19,122,102,66,94,10,22,50,46,154,134,162,190,234,246,210,206,83,79,107,119,35,63,27,7,179,175,139,151,195,223,251,231,142,146,182,170,254,226,198,218,110,114,86,74,30,2,38,58,244,232,204,208,132,152,188,160,20,8,44,48,100,120,92,64,41,53,17,13,89,69,97,125,201,213,241,237,185,165,129,157,166,186,158,130,214,202,238,242,70,90,126,98,54,42,14,18,123,103,67,95,11,23,51,47,155,135,163,191,235,247,211,207,1,29,57,37,113,109,73,85,225,253,217,197,145,141,169,181,220,192,228,248,172,176,148,136,60,32,4,24,76,80,116,104,245,233,205,209,133,153,189,161,21,9,45,49,101,121,93,65,40,52,16,12,88,68,96,124,200,212,240,236,184,164,128,156,82,78,106,118,34,62,26,6,178,174,138,150,194,222,250,230,143,147,183,171,255,227,199,219,111,115,87,75,31,3,39,59,0,29,58,39,116,105,78,83,232,245,210,207,156,129,166,187,205,208,247,234,185,164,131,158,37,56,31,2,81,76,107,118,135,154,189,160,243,238,201,212,111,114,85,72,27,6,33,60,74,87,112,109,62,35,4,25,162,191,152,133,214,203,236,241,19,14,41,52,103,122,93,64,251,230,193,220,143,146,181,168,222,195,228,249,170,183,144,141,54,43,12,17,66,95,120,101,148,137,174,179,224,253,218,199,124,97,70,91,8,21,50,47,89,68,99,126,45,48,23,10,177,172,139,150,197,216,255,226,38,59,28,1,82,79,104,117,206,211,244,233,186,167,128,157,235,246,209,204,159,130,165,184,3,30,57,36,119,106,77,80,161,188,155,134,213,200,239,242,73,84,115,110,61,32,7,26,108,113,86,75,24,5,34,63,132,153,190,163,240,237,202,215,53,40,15,18,65,92,123,102,221,192,231,250,169,180,147,142,248,229,194,223,140,145,182,171,16,13,42,55,100,121,94,67,178,175,136,149,198,219,252,225,90,71,96,125,46,51,20,9,127,98,69,88,11,22,49,44,151,138,173,176,227,254,217,196,0,30,60,34,120,102,68,90,240,238,204,210,136,150,180,170,253,227,193,223,133,155,185,167,13,19,49,47,117,107,73,87,231,249,219,197,159,129,163,189,23,9,43,53,111,113,83,77,26,4,38,56,98,124,94,64,234,244,214,200,146,140,174,176,211,205,239,241,171,181,151,137,35,61,31,1,91,69,103,121,46,48,18,12,86,72,106,116,222,192,226,252,166,184,154,132,52,42,8,22,76,82,112,110,196,218,248,230,188,162,128,158,201,215,245,235,177,175,141,147,57,39,5,27,65,95,125,99,187,165,135,153,195,221,255,225,75,85,119,105,51,45,15,17,70,88,122,100,62,32,2,28,182,168,138,148,206,208,242,236,92,66,96,126,36,58,24,6,172,178,144,142,212,202,232,246,161,191,157,131,217,199,229,251,81,79,109,115,41,55,21,11,104,118,84,74,16,14,44,50,152,134,164,186,224,254,220,194,149,139,169,183,237,243,209,207,101,123,89,71,29,3,33,63,143,145,179,173,247,233,203,213,127,97,67,93,7,25,59,37,114,108,78,80,10,20,54,40,130,156,190,160,250,228,198,216,0,31,62,33,124,99,66,93,248,231,198,217,132,155,186,165,237,242,211,204,145,142,175,176,21,10,43,52,105,118,87,72,199,216,249,230,187,164,133,154,63,32,1,30,67,92,125,98,42,53,20,11,86,73,104,119,210,205,236,243,174,177,144,143,147,140,173,178,239,240,209,206,107,116,85,74,23,8,41,54,126,97,64,95,2,29,60,35,134,153,184,167,250,229,196,219,84,75,106,117,40,55,22,9,172,179,146,141,208,207,238,241,185,166,135,152,197,218,251,228,65,94,127,96,61,34,3,28,59,36,5,26,71,88,121,102,195,220,253,226,191,160,129,158,214,201,232,247,170,181,148,139,46,49,16,15,82,77,108,115,252,227,194,221,128,159,190,161,4,27,58,37,120,103,70,89,17,14,47,48,109,114,83,76,233,246,215,200,149,138,171,180,168,183,150,137,212,203,234,245,80,79,110,113,44,51,18,13,69,90,123,100,57,38,7,24,189,162,131,156,193,222,255,224,111,112,81,78,19,12,45,50,151,136,169,182,235,244,213,202,130,157,188,163,254,225,192,223,122,101,68,91,6,25,56,39,0,32,64,96,128,160,192,224,29,61,93,125,157,189,221,253,58,26,122,90,186,154,250,218,39,7,103,71,167,135,231,199,116,84,52,20,244,212,180,148,105,73,41,9,233,201,169,137,78,110,14,46,206,238,142,174,83,115,19,51,211,243,147,179,232,200,168,136,104,72,40,8,245,213,181,149,117,85,53,21,210,242,146,178,82,114,18,50,207,239,143,175,79,111,15,47,156,188,220,252,28,60,92,124,129,161,193,225,1,33,65,97,166,134,230,198,38,6,102,70,187,155,251,219,59,27,123,91,205,237,141,173,77,109,13,45,208,240,144,176,80,112,16,48,247,215,183,151,119,87,55,23,234,202,170,138,106,74,42,10,185,153,249,217,57,25,121,89,164,132,228,196,36,4,100,68,131,163,195,227,3,35,67,99,158,190,222,254,30,62,94,126,37,5,101,69,165,133,229,197,56,24,120,88,184,152,248,216,31,63,95,127,159,191,223,255,2,34,66,98,130,162,194,226,81,113,17,49,209,241,145,177,76,108,12,44,204,236,140,172,107,75,43,11,235,203,171,139,118,86,54,22,246,214,182,150,0,33,66,99,132,165,198,231,21,52,87,118,145,176,211,242,42,11,104,73,174,143,236,205,63,30,125,92,187,154,249,216,84,117,22,55,208,241,146,179,65,96,3,34,197,228,135,166,126,95,60,29,250,219,184,153,107,74,41,8,239,206,173,140,168,137,234,203,44,13,110,79,189,156,255,222,57,24,123,90,130,163,192,225,6,39,68,101,151,182,213,244,19,50,81,112,252,221,190,159,120,89,58,27,233,200,171,138,109,76,47,14,214,247,148,181,82,115,16,49,195,226,129,160,71,102,5,36,77,108,15,46,201,232,139,170,88,121,26,59,220,253,158,191,103,70,37,4,227,194,161,128,114,83,48,17,246,215,180,149,25,56,91,122,157,188,223,254,12,45,78,111,136,169,202,235,51,18,113,80,183,150,245,212,38,7,100,69,162,131,224,193,229,196,167,134,97,64,35,2,240,209,178,147,116,85,54,23,207,238,141,172,75,106,9,40,218,251,152,185,94,127,28,61,177,144,243,210,53,20,119,86,164,133,230,199,32,1,98,67,155,186,217,248,31,62,93,124,142,175,204,237,10,43,72,105,0,34,68,102,136,170,204,238,13,47,73,107,133,167,193,227,26,56,94,124,146,176,214,244,23,53,83,113,159,189,219,249,52,22,112,82,188,158,248,218,57,27,125,95,177,147,245,215,46,12,106,72,166,132,226,192,35,1,103,69,171,137,239,205,104,74,44,14,224,194,164,134,101,71,33,3,237,207,169,139,114,80,54,20,250,216,190,156,127,93,59,25,247,213,179,145,92,126,24,58,212,246,144,178,81,115,21,55,217,251,157,191,70,100,2,32,206,236,138,168,75,105,15,45,195,225,135,165,208,242,148,182,88,122,28,62,221,255,153,187,85,119,17,51,202,232,142,172,66,96,6,36,199,229,131,161,79,109,11,41,228,198,160,130,108,78,40,10,233,203,173,143,97,67,37,7,254,220,186,152,118,84,50,16,243,209,183,149,123,89,63,29,184,154,252,222,48,18,116,86,181,151,241,211,61,31,121,91,162,128,230,196,42,8,110,76,175,141,235,201,39,5,99,65,140,174,200,234,4,38,64,98,129,163,197,231,9,43,77,111,150,180,210,240,30,60,90,120,155,185,223,253,19,49,87,117,0,35,70,101,140,175,202,233,5,38,67,96,137,170,207,236,10,41,76,111,134,165,192,227,15,44,73,106,131,160,197,230,20,55,82,113,152,187,222,253,17,50,87,116,157,190,219,248,30,61,88,123,146,177,212,247,27,56,93,126,151,180,209,242,40,11,110,77,164,135,226,193,45,14,107,72,161,130,231,196,34,1,100,71,174,141,232,203,39,4,97,66,171,136,237,206,60,31,122,89,176,147,246,213,57,26,127,92,181,150,243,208,54,21,112,83,186,153,252,223,51,16,117,86,191,156,249,218,80,115,22,53,220,255,154,185,85,118,19,48,217,250,159,188,90,121,28,63,214,245,144,179,95,124,25,58,211,240,149,182,68,103,2,33,200,235,142,173,65,98,7,36,205,238,139,168,78,109,8,43,194,225,132,167,75,104,13,46,199,228,129,162,120,91,62,29,244,215,178,145,125,94,59,24,241,210,183,148,114,81,52,23,254,221,184,155,119,84,49,18,251,216,189,158,108,79,42,9,224,195,166,133,105,74,47,12,229,198,163,128,102,69,32,3,234,201,172,143,99,64,37,6,239,204,169,138,0,36,72,108,144,180,216,252,61,25,117,81,173,137,229,193,122,94,50,22,234,206,162,134,71,99,15,43,215,243,159,187,244,208,188,152,100,64,44,8,201,237,129,165,89,125,17,53,142,170,198,226,30,58,86,114,179,151,251,223,35,7,107,79,245,209,189,153,101,65,45,9,200,236,128,164,88,124,16,52,143,171,199,227,31,59,87,115,178,150,250,222,34,6,106,78,1,37,73,109,145,181,217,253,60,24,116,80,172,136,228,192,123,95,51,23,235,207,163,135,70,98,14,42,214,242,158,186,247,211,191,155,103,67,47,11,202,238,130,166,90,126,18,54,141,169,197,225,29,57,85,113,176,148,248,220,32,4,104,76,3,39,75,111,147,183,219,255,62,26,118,82,174,138,230,194,121,93,49,21,233,205,161,133,68,96,12,40,212,240,156,184,2,38,74,110,146,182,218,254,63,27,119,83,175,139,231,195,120,92,48,20,232,204,160,132,69,97,13,41,213,241,157,185,246,210,190,154,102,66,46,10,203,239,131,167,91,127,19,55,140,168,196,224,28,56,84,112,177,149,249,221,33,5,105,77,0,37,74,111,148,177,222,251,53,16,127,90,161,132,235,206,106,79,32,5,254,219,180,145,95,122,21,48,203,238,129,164,212,241,158,187,64,101,10,47,225,196,171,142,117,80,63,26,190,155,244,209,42,15,96,69,139,174,193,228,31,58,85,112,181,144,255,218,33,4,107,78,128,165,202,239,20,49,94,123,223,250,149,176,75,110,1,36,234,207,160,133,126,91,52,17,97,68,43,14,245,208,191,154,84,113,30,59,192,229,138,175,11,46,65,100,159,186,213,240,62,27,116,81,170,143,224,197,119,82,61,24,227,198,169,140,66,103,8,45,214,243,156,185,29,56,87,114,137,172,195,230,40,13,98,71,188,153,246,211,163,134,233,204,55,18,125,88,150,179,220,249,2,39,72,109,201,236,131,166,93,120,23,50,252,217,182,147,104,77,34,7,194,231,136,173,86,115,28,57,247,210,189,152,99,70,41,12,168,141,226,199,60,25,118,83,157,184,215,242,9,44,67,102,22,51,92,121,130,167,200,237,35,6,105,76,183,146,253,216,124,89,54,19,232,205,162,135,73,108,3,38,221,248,151,178,0,38,76,106,152,190,212,242,45,11,97,71,181,147,249,223,90,124,22,48,194,228,142,168,119,81,59,29,239,201,163,133,180,146,248,222,44,10,96,70,153,191,213,243,1,39,77,107,238,200,162,132,118,80,58,28,195,229,143,169,91,125,23,49,117,83,57,31,237,203,161,135,88,126,20,50,192,230,140,170,47,9,99,69,183,145,251,221,2,36,78,104,154,188,214,240,193,231,141,171,89,127,21,51,236,202,160,134,116,82,56,30,155,189,215,241,3,37,79,105,182,144,250,220,46,8,98,68,234,204,166,128,114,84,62,24,199,225,139,173,95,121,19,53,176,150,252,218,40,14,100,66,157,187,209,247,5,35,73,111,94,120,18,52,198,224,138,172,115,85,63,25,235,205,167,129,4,34,72,110,156,186,208,246,41,15,101,67,177,151,253,219,159,185,211,245,7,33,75,109,178,148,254,216,42,12,102,64,197,227,137,175,93,123,17,55,232,206,164,130,112,86,60,26,43,13,103,65,179,149,255,217,6,32,74,108,158,184,210,244,113,87,61,27,233,207,165,131,92,122,16,54,196,226,136,174,0,39,78,105,156,187,210,245,37,2,107,76,185,158,247,208,74,109,4,35,214,241,152,191,111,72,33,6,243,212,189,154,148,179,218,253,8,47,70,97,177,150,255,216,45,10,99,68,222,249,144,183,66,101,12,43,251,220,181,146,103,64,41,14,53,18,123,92,169,142,231,192,16,55,94,121,140,171,194,229,127,88,49,22,227,196,173,138,90,125,20,51,198,225,136,175,161,134,239,200,61,26,115,84,132,163,202,237,24,63,86,113,235,204,165,130,119,80,57,30,206,233,128,167,82,117,28,59,106,77,36,3,246,209,184,159,79,104,1,38,211,244,157,186,32,7,110,73,188,155,242,213,5,34,75,108,153,190,215,240,254,217,176,151,98,69,44,11,219,252,149,178,71,96,9,46,180,147,250,221,40,15,102,65,145,182,223,248,13,42,67,100,95,120,17,54,195,228,141,170,122,93,52,19,230,193,168,143,21,50,91,124,137,174,199,224,48,23,126,89,172,139,226,197,203,236,133,162,87,112,25,62,238,201,160,135,114,85,60,27,129,166,207,232,29,58,83,116,164,131,234,205,56,31,118,81,0,40,80,120,160,136,240,216,93,117,13,37,253,213,173,133,186,146,234,194,26,50,74,98,231,207,183,159,71,111,23,63,105,65,57,17,201,225,153,177,52,28,100,76,148,188,196,236,211,251,131,171,115,91,35,11,142,166,222,246,46,6,126,86,210,250,130,170,114,90,34,10,143,167,223,247,47,7,127,87,104,64,56,16,200,224,152,176,53,29,101,77,149,189,197,237,187,147,235,195,27,51,75,99,230,206,182,158,70,110,22,62,1,41,81,121,161,137,241,217,92,116,12,36,252,212,172,132,185,145,233,193,25,49,73,97,228,204,180,156,68,108,20,60,3,43,83,123,163,139,243,219,94,118,14,38,254,214,174,134,208,248,128,168,112,88,32,8,141,165,221,245,45,5,125,85,106,66,58,18,202,226,154,178,55,31,103,79,151,191,199,239,107,67,59,19,203,227,155,179,54,30,102,78,150,190,198,238,209,249,129,169,113,89,33,9,140,164,220,244,44,4,124,84,2,42,82,122,162,138,242,218,95,119,15,39,255,215,175,135,184,144,232,192,24,48,72,96,229,205,181,157,69,109,21,61,0,41,82,123,164,141,246,223,85,124,7,46,241,216,163,138,170,131,248,209,14,39,92,117,255,214,173,132,91,114,9,32,73,96,27,50,237,196,191,150,28,53,78,103,184,145,234,195,227,202,177,152,71,110,21,60,182,159,228,205,18,59,64,105,146,187,192,233,54,31,100,77,199,238,149,188,99,74,49,24,56,17,106,67,156,181,206,231,109,68,63,22,201,224,155,178,219,242,137,160,127,86,45,4,142,167,220,245,42,3,120,81,113,88,35,10,213,252,135,174,36,13,118,95,128,169,210,251,57,16,107,66,157,180,207,230,108,69,62,23,200,225,154,179,147,186,193,232,55,30,101,76,198,239,148,189,98,75,48,25,112,89,34,11,212,253,134,175,37,12,119,94,129,168,211,250,218,243,136,161,126,87,44,5,143,166,221,244,43,2,121,80,171,130,249,208,15,38,93,116,254,215,172,133,90,115,8,33,1,40,83,122,165,140,247,222,84,125,6,47,240,217,162,139,226,203,176,153,70,111,20,61,183,158,229,204,19,58,65,104,72,97,26,51,236,197,190,151,29,52,79,102,185,144,235,194,0,42,84,126,168,130,252,214,77,103,25,51,229,207,177,155,154,176,206,228,50,24,102,76,215,253,131,169,127,85,43,1,41,3,125,87,129,171,213,255,100,78,48,26,204,230,152,178,179,153,231,205,27,49,79,101,254,212,170,128,86,124,2,40,82,120,6,44,250,208,174,132,31,53,75,97,183,157,227,201,200,226,156,182,96,74,52,30,133,175,209,251,45,7,121,83,123,81,47,5,211,249,135,173,54,28,98,72,158,180,202,224,225,203,181,159,73,99,29,55,172,134,248,210,4,46,80,122,164,142,240,218,12,38,88,114,233,195,189,151,65,107,21,63,62,20,106,64,150,188,194,232,115,89,39,13,219,241,143,165,141,167,217,243,37,15,113,91,192,234,148,190,104,66,60,22,23,61,67,105,191,149,235,193,90,112,14,36,242,216,166,140,246,220,162,136,94,116,10,32,187,145,239,197,19,57,71,109,108,70,56,18,196,238,144,186,33,11,117,95,137,163,221,247,223,245,139,161,119,93,35,9,146,184,198,236,58,16,110,68,69,111,17,59,237,199,185,147,8,34,92,118,160,138,244,222,0,43,86,125,172,135,250,209,69,110,19,56,233,194,191,148,138,161,220,247,38,13,112,91,207,228,153,178,99,72,53,30,9,34,95,116,165,142,243,216,76,103,26,49,224,203,182,157,131,168,213,254,47,4,121,82,198,237,144,187,106,65,60,23,18,57,68,111,190,149,232,195,87,124,1,42,251,208,173,134,152,179,206,229,52,31,98,73,221,246,139,160,113,90,39,12,27,48,77,102,183,156,225,202,94,117,8,35,242,217,164,143,145,186,199,236,61,22,107,64,212,255,130,169,120,83,46,5,36,15,114,89,136,163,222,245,97,74,55,28,205,230,155,176,174,133,248,211,2,41,84,127,235,192,189,150,71,108,17,58,45,6,123,80,129,170,215,252,104,67,62,21,196,239,146,185,167,140,241,218,11,32,93,118,226,201,180,159,78,101,24,51,54,29,96,75,154,177,204,231,115,88,37,14,223,244,137,162,188,151,234,193,16,59,70,109,249,210,175,132,85,126,3,40,63,20,105,66,147,184,197,238,122,81,44,7,214,253,128,171,181,158,227,200,25,50,79,100,240,219,166,141,92,119,10,33,0,44,88,116,176,156,232,196,125,81,37,9,205,225,149,185,250,214,162,142,74,102,18,62,135,171,223,243,55,27,111,67,233,197,177,157,89,117,1,45,148,184,204,224,36,8,124,80,19,63,75,103,163,143,251,215,110,66,54,26,222,242,134,170,207,227,151,187,127,83,39,11,178,158,234,198,2,46,90,118,53,25,109,65,133,169,221,241,72,100,16,60,248,212,160,140,38,10,126,82,150,186,206,226,91,119,3,47,235,199,179,159,220,240,132,168,108,64,52,24,161,141,249,213,17,61,73,101,131,175,219,247,51,31,107,71,254,210,166,138,78,98,22,58,121,85,33,13,201,229,145,189,4,40,92,112,180,152,236,192,106,70,50,30,218,246,130,174,23,59,79,99,167,139,255,211,144,188,200,228,32,12,120,84,237,193,181,153,93,113,5,41,76,96,20,56,252,208,164,136,49,29,105,69,129,173,217,245,182,154,238,194,6,42,94,114,203,231,147,191,123,87,35,15,165,137,253,209,21,57,77,97,216,244,128,172,104,68,48,28,95,115,7,43,239,195,183,155,34,14,122,86,146,190,202,230,0,45,90,119,180,153,238,195,117,88,47,2,193,236,155,182,234,199,176,157,94,115,4,41,159,178,197,232,43,6,113,92,201,228,147,190,125,80,39,10,188,145,230,203,8,37,82,127,35,14,121,84,151,186,205,224,86,123,12,33,226,207,184,149,143,162,213,248,59,22,97,76,250,215,160,141,78,99,20,57,101,72,63,18,209,252,139,166,16,61,74,103,164,137,254,211,70,107,28,49,242,223,168,133,51,30,105,68,135,170,221,240,172,129,246,219,24,53,66,111,217,244,131,174,109,64,55,26,3,46,89,116,183,154,237,192,118,91,44,1,194,239,152,181,233,196,179,158,93,112,7,42,156,177,198,235,40,5,114,95,202,231,144,189,126,83,36,9,191,146,229,200,11,38,81,124,32,13,122,87,148,185,206,227,85,120,15,34,225,204,187,150,140,161,214,251,56,21,98,79,249,212,163,142,77,96,23,58,102,75,60,17,210,255,136,165,19,62,73,100,167,138,253,208,69,104,31,50,241,220,171,134,48,29,106,71,132,169,222,243,175,130,245,216,27,54,65,108,218,247,128,173,110,67,52,25,0,46,92,114,184,150,228,202,109,67,49,31,213,251,137,167,218,244,134,168,98,76,62,16,183,153,235,197,15,33,83,125,169,135,245,219,17,63,77,99,196,234,152,182,124,82,32,14,115,93,47,1,203,229,151,185,30,48,66,108,166,136,250,212,79,97,19,61,247,217,171,133,34,12,126,80,154,180,198,232,149,187,201,231,45,3,113,95,248,214,164,138,64,110,28,50,230,200,186,148,94,112,2,44,139,165,215,249,51,29,111,65,60,18,96,78,132,170,216,246,81,127,13,35,233,199,181,155,158,176,194,236,38,8,122,84,243,221,175,129,75,101,23,57,68,106,24,54,252,210,160,142,41,7,117,91,145,191,205,227,55,25,107,69,143,161,211,253,90,116,6,40,226,204,190,144,237,195,177,159,85,123,9,39,128,174,220,242,56,22,100,74,209,255,141,163,105,71,53,27,188,146,224,206,4,42,88,118,11,37,87,121,179,157,239,193,102,72,58,20,222,240,130,172,120,86,36,10,192,238,156,178,21,59,73,103,173,131,241,223,162,140,254,208,26,52,70,104,207,225,147,189,119,89,43,5,0,47,94,113,188,147,226,205,101,74,59,20,217,246,135,168,202,229,148,187,118,89,40,7,175,128,241,222,19,60,77,98,137,166,215,248,53,26,107,68,236,195,178,157,80,127,14,33,67,108,29,50,255,208,161,142,38,9,120,87,154,181,196,235,15,32,81,126,179,156,237,194,106,69,52,27,214,249,136,167,197,234,155,180,121,86,39,8,160,143,254,209,28,51,66,109,134,169,216,247,58,21,100,75,227,204,189,146,95,112,1,46,76,99,18,61,240,223,174,129,41,6,119,88,149,186,203,228,30,49,64,111,162,141,252,211,123,84,37,10,199,232,153,182,212,251,138,165,104,71,54,25,177,158,239,192,13,34,83,124,151,184,201,230,43,4,117,90,242,221,172,131,78,97,16,63,93,114,3,44,225,206,191,144,56,23,102,73,132,171,218,245,17,62,79,96,173,130,243,220,116,91,42,5,200,231,150,185,219,244,133,170,103,72,57,22,190,145,224,207,2,45,92,115,152,183,198,233,36,11,122,85,253,210,163,140,65,110,31,48,82,125,12,35,238,193,176,159,55,24,105,70,139,164,213,250,0,48,96,80,192,240,160,144,157,173,253,205,93,109,61,13,39,23,71,119,231,215,135,183,186,138,218,234,122,74,26,42,78,126,46,30,142,190,238,222,211,227,179,131,19,35,115,67,105,89,9,57,169,153,201,249,244,196,148,164,52,4,84,100,156,172,252,204,92,108,60,12,1,49,97,81,193,241,161,145,187,139,219,235,123,75,27,43,38,22,70,118,230,214,134,182,210,226,178,130,18,34,114,66,79,127,47,31,143,191,239,223,245,197,149,165,53,5,85,101,104,88,8,56,168,152,200,248,37,21,69,117,229,213,133,181,184,136,216,232,120,72,24,40,2,50,98,82,194,242,162,146,159,175,255,207,95,111,63,15,107,91,11,59,171,155,203,251,246,198,150,166,54,6,86,102,76,124,44,28,140,188,236,220,209,225,177,129,17,33,113,65,185,137,217,233,121,73,25,41,36,20,68,116,228,212,132,180,158,174,254,206,94,110,62,14,3,51,99,83,195,243,163,147,247,199,151,167,55,7,87,103,106,90,10,58,170,154,202,250,208,224,176,128,16,32,112,64,77,125,45,29,141,189,237,221,0,49,98,83,196,245,166,151,149,164,247,198,81,96,51,2,55,6,85,100,243,194,145,160,162,147,192,241,102,87,4,53,110,95,12,61,170,155,200,249,251,202,153,168,63,14,93,108,89,104,59,10,157,172,255,206,204,253,174,159,8,57,106,91,220,237,190,143,24,41,122,75,73,120,43,26,141,188,239,222,235,218,137,184,47,30,77,124,126,79,28,45,186,139,216,233,178,131,208,225,118,71,20,37,39,22,69,116,227,210,129,176,133,180,231,214,65,112,35,18,16,33,114,67,212,229,182,135,165,148,199,246,97,80,3,50,48,1,82,99,244,197,150,167,146,163,240,193,86,103,52,5,7,54,101,84,195,242,161,144,203,250,169,152,15,62,109,92,94,111,60,13,154,171,248,201,252,205,158,175,56,9,90,107,105,88,11,58,173,156,207,254,121,72,27,42,189,140,223,238,236,221,142,191,40,25,74,123,78,127,44,29,138,187,232,217,219,234,185,136,31,46,125,76,23,38,117,68,211,226,177,128,130,179,224,209,70,119,36,21,32,17,66,115,228,213,134,183,181,132,215,230,113,64,19,34,0,50,100,86,200,250,172,158,141,191,233,219,69,119,33,19,7,53,99,81,207,253,171,153,138,184,238,220,66,112,38,20,14,60,106,88,198,244,162,144,131,177,231,213,75,121,47,29,9,59,109,95,193,243,165,151,132,182,224,210,76,126,40,26,28,46,120,74,212,230,176,130,145,163,245,199,89,107,61,15,27,41,127,77,211,225,183,133,150,164,242,192,94,108,58,8,18,32,118,68,218,232,190,140,159,173,251,201,87,101,51,1,21,39,113,67,221,239,185,139,152,170,252,206,80,98,52,6,56,10,92,110,240,194,148,166,181,135,209,227,125,79,25,43,63,13,91,105,247,197,147,161,178,128,214,228,122,72,30,44,54,4,82,96,254,204,154,168,187,137,223,237,115,65,23,37,49,3,85,103,249,203,157,175,188,142,216,234,116,70,16,34,36,22,64,114,236,222,136,186,169,155,205,255,97,83,5,55,35,17,71,117,235,217,143,189,174,156,202,248,102,84,2,48,42,24,78,124,226,208,134,180,167,149,195,241,111,93,11,57,45,31,73,123,229,215,129,179,160,146,196,246,104,90,12,62,0,51,102,85,204,255,170,153,133,182,227,208,73,122,47,28,23,36,113,66,219,232,189,142,146,161,244,199,94,109,56,11,46,29,72,123,226,209,132,183,171,152,205,254,103,84,1,50,57,10,95,108,245,198,147,160,188,143,218,233,112,67,22,37,92,111,58,9,144,163,246,197,217,234,191,140,21,38,115,64,75,120,45,30,135,180,225,210,206,253,168,155,2,49,100,87,114,65,20,39,190,141,216,235,247,196,145,162,59,8,93,110,101,86,3,48,169,154,207,252,224,211,134,181,44,31,74,121,184,139,222,237,116,71,18,33,61,14,91,104,241,194,151,164,175,156,201,250,99,80,5,54,42,25,76,127,230,213,128,179,150,165,240,195,90,105,60,15,19,32,117,70,223,236,185,138,129,178,231,212,77,126,43,24,4,55,98,81,200,251,174,157,228,215,130,177,40,27,78,125,97,82,7,52,173,158,203,248,243,192,149,166,63,12,89,106,118,69,16,35,186,137,220,239,202,249,172,159,6,53,96,83,79,124,41,26,131,176,229,214,221,238,187,136,17,34,119,68,88,107,62,13,148,167,242,193,0,52,104,92,208,228,184,140,189,137,213,225,109,89,5,49,103,83,15,59,183,131,223,235,218,238,178,134,10,62,98,86,206,250,166,146,30,42,118,66,115,71,27,47,163,151,203,255,169,157,193,245,121,77,17,37,20,32,124,72,196,240,172,152,129,181,233,221,81,101,57,13,60,8,84,96,236,216,132,176,230,210,142,186,54,2,94,106,91,111,51,7,139,191,227,215,79,123,39,19,159,171,247,195,242,198,154,174,34,22,74,126,40,28,64,116,248,204,144,164,149,161,253,201,69,113,45,25,31,43,119,67,207,251,167,147,162,150,202,254,114,70,26,46,120,76,16,36,168,156,192,244,197,241,173,153,21,33,125,73,209,229,185,141,1,53,105,93,108,88,4,48,188,136,212,224,182,130,222,234,102,82,14,58,11,63,99,87,219,239,179,135,158,170,246,194,78,122,38,18,35,23,75,127,243,199,155,175,249,205,145,165,41,29,65,117,68,112,44,24,148,160,252,200,80,100,56,12,128,180,232,220,237,217,133,177,61,9,85,97,55,3,95,107,231,211,143,187,138,190,226,214,90,110,50,6,0,53,106,95,212,225,190,139,181,128,223,234,97,84,11,62,119,66,29,40,163,150,201,252,194,247,168,157,22,35,124,73,238,219,132,177,58,15,80,101,91,110,49,4,143,186,229,208,153,172,243,198,77,120,39,18,44,25,70,115,248,205,146,167,193,244,171,158,21,32,127,74,116,65,30,43,160,149,202,255,182,131,220,233,98,87,8,61,3,54,105,92,215,226,189,136,47,26,69,112,251,206,145,164,154,175,240,197,78,123,36,17,88,109,50,7,140,185,230,211,237,216,135,178,57,12,83,102,159,170,245,192,75,126,33,20,42,31,64,117,254,203,148,161,232,221,130,183,60,9,86,99,93,104,55,2,137,188,227,214,113,68,27,46,165,144,207,250,196,241,174,155,16,37,122,79,6,51,108,89,210,231,184,141,179,134,217,236,103,82,13,56,94,107,52,1,138,191,224,213,235,222,129,180,63,10,85,96,41,28,67,118,253,200,151,162,156,169,246,195,72,125,34,23,176,133,218,239,100,81,14,59,5,48,111,90,209,228,187,142,199,242,173,152,19,38,121,76,114,71,24,45,166,147,204,249,0,54,108,90,216,238,180,130,173,155,193,247,117,67,25,47,71,113,43,29,159,169,243,197,234,220,134,176,50,4,94,104,142,184,226,212,86,96,58,12,35,21,79,121,251,205,151,161,201,255,165,147,17,39,125,75,100,82,8,62,188,138,208,230,1,55,109,91,217,239,181,131,172,154,192,246,116,66,24,46,70,112,42,28,158,168,242,196,235,221,135,177,51,5,95,105,143,185,227,213,87,97,59,13,34,20,78,120,250,204,150,160,200,254,164,146,16,38,124,74,101,83,9,63,189,139,209,231,2,52,110,88,218,236,182,128,175,153,195,245,119,65,27,45,69,115,41,31,157,171,241,199,232,222,132,178,48,6,92,106,140,186,224,214,84,98,56,14,33,23,77,123,249,207,149,163,203,253,167,145,19,37,127,73,102,80,10,60,190,136,210,228,3,53,111,89,219,237,183,129,174,152,194,244,118,64,26,44,68,114,40,30,156,170,240,198,233,223,133,179,49,7,93,107,141,187,225,215,85,99,57,15,32,22,76,122,248,206,148,162,202,252,166,144,18,36,126,72,103,81,11,61,191,137,211,229,0,55,110,89,220,235,178,133,165,146,203,252,121,78,23,32,87,96,57,14,139,188,229,210,242,197,156,171,46,25,64,119,174,153,192,247,114,69,28,43,11,60,101,82,215,224,185,142,249,206,151,160,37,18,75,124,92,107,50,5,128,183,238,217,65,118,47,24,157,170,243,196,228,211,138,189,56,15,86,97,22,33,120,79,202,253,164,147,179,132,221,234,111,88,1,54,239,216,129,182,51,4,93,106,74,125,36,19,150,161,248,207,184,143,214,225,100,83,10,61,29,42,115,68,193,246,175,152,130,181,236,219,94,105,48,7,39,16,73,126,251,204,149,162,213,226,187,140,9,62,103,80,112,71,30,41,172,155,194,245,44,27,66,117,240,199,158,169,137,190,231,208,85,98,59,12,123,76,21,34,167,144,201,254,222,233,176,135,2,53,108,91,195,244,173,154,31,40,113,70,102,81,8,63,186,141,212,227,148,163,250,205,72,127,38,17,49,6,95,104,237,218,131,180,109,90,3,52,177,134,223,232,200,255,166,145,20,35,122,77,58,13,84,99,230,209,136,191,159,168,241,198,67,116,45,26,0,56,112,72,224,216,144,168,221,229,173,149,61,5,77,117,167,159,215,239,71,127,55,15,122,66,10,50,154,162,234,210,83,107,35,27,179,139,195,251,142,182,254,198,110,86,30,38,244,204,132,188,20,44,100,92,41,17,89,97,201,241,185,129,166,158,214,238,70,126,54,14,123,67,11,51,155,163,235,211,1,57,113,73,225,217,145,169,220,228,172,148,60,4,76,116,245,205,133,189,21,45,101,93,40,16,88,96,200,240,184,128,82,106,34,26,178,138,194,250,143,183,255,199,111,87,31,39,81,105,33,25,177,137,193,249,140,180,252,196,108,84,28,36,246,206,134,190,22,46,102,94,43,19,91,99,203,243,187,131,2,58,114,74,226,218,146,170,223,231,175,151,63,7,79,119,165,157,213,237,69,125,53,13,120,64,8,48,152,160,232,208,247,207,135,191,23,47,103,95,42,18,90,98,202,242,186,130,80,104,32,24,176,136,192,248,141,181,253,197,109,85,29,37,164,156,212,236,68,124,52,12,121,65,9,49,153,161,233,209,3,59,115,75,227,219,147,171,222,230,174,150,62,6,78,118,0,57,114,75,228,221,150,175,213,236,167,158,49,8,67,122,183,142,197,252,83,106,33,24,98,91,16,41,134,191,244,205,115,74,1,56,151,174,229,220,166,159,212,237,66,123,48,9,196,253,182,143,32,25,82,107,17,40,99,90,245,204,135,190,230,223,148,173,2,59,112,73,51,10,65,120,215,238,165,156,81,104,35,26,181,140,199,254,132,189,246,207,96,89,18,43,149,172,231,222,113,72,3,58,64,121,50,11,164,157,214,239,34,27,80,105,198,255,180,141,247,206,133,188,19,42,97,88,209,232,163,154,53,12,71,126,4,61,118,79,224,217,146,171,102,95,20,45,130,187,240,201,179,138,193,248,87,110,37,28,162,155,208,233,70,127,52,13,119,78,5,60,147,170,225,216,21,44,103,94,241,200,131,186,192,249,178,139,36,29,86,111,55,14,69,124,211,234,161,152,226,219,144,169,6,63,116,77,128,185,242,203,100,93,22,47,85,108,39,30,177,136,195,250,68,125,54,15,160,153,210,235,145,168,227,218,117,76,7,62,243,202,129,184,23,46,101,92,38,31,84,109,194,251,176,137,0,58,116,78,232,210,156,166,205,247,185,131,37,31,81,107,135,189,243,201,111,85,27,33,74,112,62,4,162,152,214,236,19,41,103,93,251,193,143,181,222,228,170,144,54,12,66,120,148,174,224,218,124,70,8,50,89,99,45,23,177,139,197,255,38,28,82,104,206,244,186,128,235,209,159,165,3,57,119,77,161,155,213,239,73,115,61,7,108,86,24,34,132,190,240,202,53,15,65,123,221,231,169,147,248,194,140,182,16,42,100,94,178,136,198,252,90,96,46,20,127,69,11,49,151,173,227,217,76,118,56,2,164,158,208,234,129,187,245,207,105,83,29,39,203,241,191,133,35,25,87,109,6,60,114,72,238,212,154,160,95,101,43,17,183,141,195,249,146,168,230,220,122,64,14,52,216,226,172,150,48,10,68,126,21,47,97,91,253,199,137,179,106,80,30,36,130,184,246,204,167,157,211,233,79,117,59,1,237,215,153,163,5,63,113,75,32,26,84,110,200,242,188,134,121,67,13,55,145,171,229,223,180,142,192,250,92,102,40,18,254,196,138,176,22,44,98,88,51,9,71,125,219,225,175,149,0,59,118,77,236,215,154,161,197,254,179,136,41,18,95,100,151,172,225,218,123,64,13,54,82,105,36,31,190,133,200,243,51,8,69,126,223,228,169,146,246,205,128,187,26,33,108,87,164,159,210,233,72,115,62,5,97,90,23,44,141,182,251,192,102,93,16,43,138,177,252,199,163,152,213,238,79,116,57,2,241,202,135,188,29,38,107,80,52,15,66,121,216,227,174,149,85,110,35,24,185,130,207,244,144,171,230,221,124,71,10,49,194,249,180,143,46,21,88,99,7,60,113,74,235,208,157,166,204,247,186,129,32,27,86,109,9,50,127,68,229,222,147,168,91,96,45,22,183,140,193,250,158,165,232,211,114,73,4,63,255,196,137,178,19,40,101,94,58,1,76,119,214,237,160,155,104,83,30,37,132,191,242,201,173,150,219,224,65,122,55,12,170,145,220,231,70,125,48,11,111,84,25,34,131,184,245,206,61,6,75,112,209,234,167,156,248,195,142,181,20,47,98,89,153,162,239,212,117,78,3,56,92,103,42,17,176,139,198,253,14,53,120,67,226,217,148,175,203,240,189,134,39,28,81,106,0,60,120,68,240,204,136,180,253,193,133,185,13,49,117,73,231,219,159,163,23,43,111,83,26,38,98,94,234,214,146,174,211,239,171,151,35,31,91,103,46,18,86,106,222,226,166,154,52,8,76,112,196,248,188,128,201,245,177,141,57,5,65,125,187,135,195,255,75,119,51,15,70,122,62,2,182,138,206,242,92,96,36,24,172,144,212,232,161,157,217,229,81,109,41,21,104,84,16,44,152,164,224,220,149,169,237,209,101,89,29,33,143,179,247,203,127,67,7,59,114,78,10,54,130,190,250,198,107,87,19,47,155,167,227,223,150,170,238,210,102,90,30,34,140,176,244,200,124,64,4,56,113,77,9,53,129,189,249,197,184,132,192,252,72,116,48,12,69,121,61,1,181,137,205,241,95,99,39,27,175,147,215,235,162,158,218,230,82,110,42,22,208,236,168,148,32,28,88,100,45,17,85,105,221,225,165,153,55,11,79,115,199,251,191,131,202,246,178,142,58,6,66,126,3,63,123,71,243,207,139,183,254,194,134,186,14,50,118,74,228,216,156,160,20,40,108,80,25,37,97,93,233,213,145,173,0,61,122,71,244,201,142,179,245,200,143,178,1,60,123,70,247,202,141,176,3,62,121,68,2,63,120,69,246,203,140,177,243,206,137,180,7,58,125,64,6,59,124,65,242,207,136,181,4,57,126,67,240,205,138,183,241,204,139,182,5,56,127,66,251,198,129,188,15,50,117,72,14,51,116,73,250,199,128,189,12,49,118,75,248,197,130,191,249,196,131,190,13,48,119,74,8,53,114,79,252,193,134,187,253,192,135,186,9,52,115,78,255,194,133,184,11,54,113,76,10,55,112,77,254,195,132,185,235,214,145,172,31,34,101,88,30,35,100,89,234,215,144,173,28,33,102,91,232,213,146,175,233,212,147,174,29,32,103,90,24,37,98,95,236,209,150,171,237,208,151,170,25,36,99,94,239,210,149,168,27,38,97,92,26,39,96,93,238,211,148,169,16,45,106,87,228,217,158,163,229,216,159,162,17,44,107,86,231,218,157,160,19,46,105,84,18,47,104,85,230,219,156,161,227,222,153,164,23,42,109,80,22,43,108,81,226,223,152,165,20,41,110,83,224,221,154,167,225,220,155,166,21,40,111,82,0,62,124,66,248,198,132,186,237,211,145,175,21,43,105,87,199,249,187,133,63,1,67,125,42,20,86,104,210,236,174,144,147,173,239,209,107,85,23,41,126,64,2,60,134,184,250,196,84,106,40,22,172,146,208,238,185,135,197,251,65,127,61,3,59,5,71,121,195,253,191,129,214,232,170,148,46,16,82,108,252,194,128,190,4,58,120,70,17,47,109,83,233,215,149,171,168,150,212,234,80,110,44,18,69,123,57,7,189,131,193,255,111,81,19,45,151,169,235,213,130,188,254,192,122,68,6,56,118,72,10,52,142,176,242,204,155,165,231,217,99,93,31,33,177,143,205,243,73,119,53,11,92,98,32,30,164,154,216,230,229,219,153,167,29,35,97,95,8,54,116,74,240,206,140,178,34,28,94,96,218,228,166,152,207,241,179,141,55,9,75,117,77,115,49,15,181,139,201,247,160,158,220,226,88,102,36,26,138,180,246,200,114,76,14,48,103,89,27,37,159,161,227,221,222,224,162,156,38,24,90,100,51,13,79,113,203,245,183,137,25,39,101,91,225,223,157,163,244,202,136,182,12,50,112,78,0,63,126,65,252,195,130,189,229,218,155,164,25,38,103,88,215,232,169,150,43,20,85,106,50,13,76,115,206,241,176,143,179,140,205,242,79,112,49,14,86,105,40,23,170,149,212,235,100,91,26,37,152,167,230,217,129,190,255,192,125,66,3,60,123,68,5,58,135,184,249,198,158,161,224,223,98,93,28,35,172,147,210,237,80,111,46,17,73,118,55,8,181,138,203,244,200,247,182,137,52,11,74,117,45,18,83,108,209,238,175,144,31,32,97,94,227,220,157,162,250,197,132,187,6,57,120,71,246,201,136,183,10,53,116,75,19,44,109,82,239,208,145,174,33,30,95,96,221,226,163,156,196,251,186,133,56,7,70,121,69,122,59,4,185,134,199,248,160,159,222,225,92,99,34,29,146,173,236,211,110,81,16,47,119,72,9,54,139,180,245,202,141,178,243,204,113,78,15,48,104,87,22,41,148,171,234,213,90,101,36,27,166,153,216,231,191,128,193,254,67,124,61,2,62,1,64,127,194,253,188,131,219,228,165,154,39,24,89,102,233,214,151,168,21,42,107,84,12,51,114,77,240,207,142,177,0,64,128,192,29,93,157,221,58,122,186,250,39,103,167,231,116,52,244,180,105,41,233,169,78,14,206,142,83,19,211,147,232,168,104,40,245,181,117,53,210,146,82,18,207,143,79,15,156,220,28,92,129,193,1,65,166,230,38,102,187,251,59,123,205,141,77,13,208,144,80,16,247,183,119,55,234,170,106,42,185,249,57,121,164,228,36,100,131,195,3,67,158,222,30,94,37,101,165,229,56,120,184,248,31,95,159,223,2,66,130,194,81,17,209,145,76,12,204,140,107,43,235,171,118,54,246,182,135,199,7,71,154,218,26,90,189,253,61,125,160,224,32,96,243,179,115,51,238,174,110,46,201,137,73,9,212,148,84,20,111,47,239,175,114,50,242,178,85,21,213,149,72,8,200,136,27,91,155,219,6,70,134,198,33,97,161,225,60,124,188,252,74,10,202,138,87,23,215,151,112,48,240,176,109,45,237,173,62,126,190,254,35,99,163,227,4,68,132,196,25,89,153,217,162,226,34,98,191,255,63,127,152,216,24,88,133,197,5,69,214,150,86,22,203,139,75,11,236,172,108,44,241,177,113,49,0,65,130,195,25,88,155,218,50,115,176,241,43,106,169,232,100,37,230,167,125,60,255,190,86,23,212,149,79,14,205,140,200,137,74,11,209,144,83,18,250,187,120,57,227,162,97,32,172,237,46,111,181,244,55,118,158,223,28,93,135,198,5,68,141,204,15,78,148,213,22,87,191,254,61,124,166,231,36,101,233,168,107,42,240,177,114,51,219,154,89,24,194,131,64,1,69,4,199,134,92,29,222,159,119,54,245,180,110,47,236,173,33,96,163,226,56,121,186,251,19,82,145,208,10,75,136,201,7,70,133,196,30,95,156,221,53,116,183,246,44,109,174,239,99,34,225,160,122,59,248,185,81,16,211,146,72,9,202,139,207,142,77,12,214,151,84,21,253,188,127,62,228,165,102,39,171,234,41,104,178,243,48,113,153,216,27,90,128,193,2,67,138,203,8,73,147,210,17,80,184,249,58,123,161,224,35,98,238,175,108,45,247,182,117,52,220,157,94,31,197,132,71,6,66,3,192,129,91,26,217,152,112,49,242,179,105,40,235,170,38,103,164,229,63,126,189,252,20,85,150,215,13,76,143,206,0,66,132,198,21,87,145,211,42,104,174,236,63,125,187,249,84,22,208,146,65,3,197,135,126,60,250,184,107,41,239,173,168,234,44,110,189,255,57,123,130,192,6,68,151,213,19,81,252,190,120,58,233,171,109,47,214,148,82,16,195,129,71,5,77,15,201,139,88,26,220,158,103,37,227,161,114,48,246,180,25,91,157,223,12,78,136,202,51,113,183,245,38,100,162,224,229,167,97,35,240,178,116,54,207,141,75,9,218,152,94,28,177,243,53,119,164,230,32,98,155,217,31,93,142,204,10,72,154,216,30,92,143,205,11,73,176,242,52,118,165,231,33,99,206,140,74,8,219,153,95,29,228,166,96,34,241,179,117,55,50,112,182,244,39,101,163,225,24,90,156,222,13,79,137,203,102,36,226,160,115,49,247,181,76,14,200,138,89,27,221,159,215,149,83,17,194,128,70,4,253,191,121,59,232,170,108,46,131,193,7,69,150,212,18,80,169,235,45,111,188,254,56,122,127,61,251,185,106,40,238,172,85,23,209,147,64,2,196,134,43,105,175,237,62,124,186,248,1,67,133,199,20,86,144,210,0,67,134,197,17,82,151,212,34,97,164,231,51,112,181,246,68,7,194,129,85,22,211,144,102,37,224,163,119,52,241,178,136,203,14,77,153,218,31,92,170,233,44,111,187,248,61,126,204,143,74,9,221,158,91,24,238,173,104,43,255,188,121,58,13,78,139,200,28,95,154,217,47,108,169,234,62,125,184,251,73,10,207,140,88,27,222,157,107,40,237,174,122,57,252,191,133,198,3,64,148,215,18,81,167,228,33,98,182,245,48,115,193,130,71,4,208,147,86,21,227,160,101,38,242,177,116,55,26,89,156,223,11,72,141,206,56,123,190,253,41,106,175,236,94,29,216,155,79,12,201,138,124,63,250,185,109,46,235,168,146,209,20,87,131,192,5,70,176,243,54,117,161,226,39,100,214,149,80,19,199,132,65,2,244,183,114,49,229,166,99,32,23,84,145,210,6,69,128,195,53,118,179,240,36,103,162,225,83,16,213,150,66,1,196,135,113,50,247,180,96,35,230,165,159,220,25,90,142,205,8,75,189,254,59,120,172,239,42,105,219,152,93,30,202,137,76,15,249,186,127,60,232,171,110,45,0,68,136,204,13,73,133,193,26,94,146,214,23,83,159,219,52,112,188,248,57,125,177,245,46,106,166,226,35,103,171,239,104,44,224,164,101,33,237,169,114,54,250,190,127,59,247,179,92,24,212,144,81,21,217,157,70,2,206,138,75,15,195,135,208,148,88,28,221,153,85,17,202,142,66,6,199,131,79,11,228,160,108,40,233,173,97,37,254,186,118,50,243,183,123,63,184,252,48,116,181,241,61,121,162,230,42,110,175,235,39,99,140,200,4,64,129,197,9,77,150,210,30,90,155,223,19,87,189,249,53,113,176,244,56,124,167,227,47,107,170,238,34,102,137,205,1,69,132,192,12,72,147,215,27,95,158,218,22,82,213,145,93,25,216,156,80,20,207,139,71,3,194,134,74,14,225,165,105,45,236,168,100,32,251,191,115,55,246,178,126,58,109,41,229,161,96,36,232,172,119,51,255,187,122,62,242,182,89,29,209,149,84,16,220,152,67,7,203,143,78,10,198,130,5,65,141,201,8,76,128,196,31,91,151,211,18,86,154,222,49,117,185,253,60,120,180,240,43,111,163,231,38,98,174,234,0,69,138,207,9,76,131,198,18,87,152,221,27,94,145,212,36,97,174,235,45,104,167,226,54,115,188,249,63,122,181,240,72,13,194,135,65,4,203,142,90,31,208,149,83,22,217,156,108,41,230,163,101,32,239,170,126,59,244,177,119,50,253,184,144,213,26,95,153,220,19,86,130,199,8,77,139,206,1,68,180,241,62,123,189,248,55,114,166,227,44,105,175,234,37,96,216,157,82,23,209,148,91,30,202,143,64,5,195,134,73,12,252,185,118,51,245,176,127,58,238,171,100,33,231,162,109,40,61,120,183,242,52,113,190,251,47,106,165,224,38,99,172,233,25,92,147,214,16,85,154,223,11,78,129,196,2,71,136,205,117,48,255,186,124,57,246,179,103,34,237,168,110,43,228,161,81,20,219,158,88,29,210,151,67,6,201,140,74,15,192,133,173,232,39,98,164,225,46,107,191,250,53,112,182,243,60,121,137,204,3,70,128,197,10,79,155,222,17,84,146,215,24,93,229,160,111,42,236,169,102,35,247,178,125,56,254,187,116,49,193,132,75,14,200,141,66,7,211,150,89,28,218,159,80,21,0,70,140,202,5,67,137,207,10,76,134,192,15,73,131,197,20,82,152,222,17,87,157,219,30,88,146,212,27,93,151,209,40,110,164,226,45,107,161,231,34,100,174,232,39,97,171,237,60,122,176,246,57,127,181,243,54,112,186,252,51,117,191,249,80,22,220,154,85,19,217,159,90,28,214,144,95,25,211,149,68,2,200,142,65,7,205,139,78,8,194,132,75,13,199,129,120,62,244,178,125,59,241,183,114,52,254,184,119,49,251,189,108,42,224,166,105,47,229,163,102,32,234,172,99,37,239,169,160,230,44,106,165,227,41,111,170,236,38,96,175,233,35,101,180,242,56,126,177,247,61,123,190,248,50,116,187,253,55,113,136,206,4,66,141,203,1,71,130,196,14,72,135,193,11,77,156,218,16,86,153,223,21,83,150,208,26,92,147,213,31,89,240,182,124,58,245,179,121,63,250,188,118,48,255,185,115,53,228,162,104,46,225,167,109,43,238,168,98,36,235,173,103,33,216,158,84,18,221,155,81,23,210,148,94,24,215,145,91,29,204,138,64,6,201,143,69,3,198,128,74,12,195,133,79,9,0,71,142,201,1,70,143,200,2,69,140,203,3,68,141,202,4,67,138,205,5,66,139,204,6,65,136,207,7,64,137,206,8,79,134,193,9,78,135,192,10,77,132,195,11,76,133,194,12,75,130,197,13,74,131,196,14,73,128,199,15,72,129,198,16,87,158,217,17,86,159,216,18,85,156,219,19,84,157,218,20,83,154,221,21,82,155,220,22,81,152,223,23,80,153,222,24,95,150,209,25,94,151,208,26,93,148,211,27,92,149,210,28,91,146,213,29,90,147,212,30,89,144,215,31,88,145,214,32,103,174,233,33,102,175,232,34,101,172,235,35,100,173,234,36,99,170,237,37,98,171,236,38,97,168,239,39,96,169,238,40,111,166,225,41,110,167,224,42,109,164,227,43,108,165,226,44,107,162,229,45,106,163,228,46,105,160,231,47,104,161,230,48,119,190,249,49,118,191,248,50,117,188,251,51,116,189,250,52,115,186,253,53,114,187,252,54,113,184,255,55,112,185,254,56,127,182,241,57,126,183,240,58,125,180,243,59,124,181,242,60,123,178,245,61,122,179,244,62,121,176,247,63,120,177,246,0,72,144,216,61,117,173,229,122,50,234,162,71,15,215,159,244,188,100,44,201,129,89,17,142,198,30,86,179,251,35,107,245,189,101,45,200,128,88,16,143,199,31,87,178,250,34,106,1,73,145,217,60,116,172,228,123,51,235,163,70,14,214,158,247,191,103,47,202,130,90,18,141,197,29,85,176,248,32,104,3,75,147,219,62,118,174,230,121,49,233,161,68,12,212,156,2,74,146,218,63,119,175,231,120,48,232,160,69,13,213,157,246,190,102,46,203,131,91,19,140,196,28,84,177,249,33,105,243,187,99,43,206,134,94,22,137,193,25,81,180,252,36,108,7,79,151,223,58,114,170,226,125,53,237,165,64,8,208,152,6,78,150,222,59,115,171,227,124,52,236,164,65,9,209,153,242,186,98,42,207,135,95,23,136,192,24,80,181,253,37,109,4,76,148,220,57,113,169,225,126,54,238,166,67,11,211,155,240,184,96,40,205,133,93,21,138,194,26,82,183,255,39,111,241,185,97,41,204,132,92,20,139,195,27,83,182,254,38,110,5,77,149,221,56,112,168,224,127,55,239,167,66,10,210,154,0,73,146,219,57,112,171,226,114,59,224,169,75,2,217,144,228,173,118,63,221,148,79,6,150,223,4,77,175,230,61,116,213,156,71,14,236,165,126,55,167,238,53,124,158,215,12,69,49,120,163,234,8,65,154,211,67,10,209,152,122,51,232,161,183,254,37,108,142,199,28,85,197,140,87,30,252,181,110,39,83,26,193,136,106,35,248,177,33,104,179,250,24,81,138,195,98,43,240,185,91,18,201,128,16,89,130,203,41,96,187,242,134,207,20,93,191,246,45,100,244,189,102,47,205,132,95,22,115,58,225,168,74,3,216,145,1,72,147,218,56,113,170,227,151,222,5,76,174,231,60,117,229,172,119,62,220,149,78,7,166,239,52,125,159,214,13,68,212,157,70,15,237,164,127,54,66,11,208,153,123,50,233,160,48,121,162,235,9,64,155,210,196,141,86,31,253,180,111,38,182,255,36,109,143,198,29,84,32,105,178,251,25,80,139,194,82,27,192,137,107,34,249,176,17,88,131,202,40,97,186,243,99,42,241,184,90,19,200,129,245,188,103,46,204,133,94,23,135,206,21,92,190,247,44,101,0,74,148,222,53,127,161,235,106,32,254,180,95,21,203,129,212,158,64,10,225,171,117,63,190,244,42,96,139,193,31,85,181,255,33,107,128,202,20,94,223,149,75,1,234,160,126,52,97,43,245,191,84,30,192,138,11,65,159,213,62,116,170,224,119,61,227,169,66,8,214,156,29,87,137,195,40,98,188,246,163,233,55,125,150,220,2,72,201,131,93,23,252,182,104,34,194,136,86,28,247,189,99,41,168,226,60,118,157,215,9,67,22,92,130,200,35,105,183,253,124,54,232,162,73,3,221,151,238,164,122,48,219,145,79,5,132,206,16,90,177,251,37,111,58,112,174,228,15,69,155,209,80,26,196,142,101,47,241,187,91,17,207,133,110,36,250,176,49,123,165,239,4,78,144,218,143,197,27,81,186,240,46,100,229,175,113,59,208,154,68,14,153,211,13,71,172,230,56,114,243,185,103,45,198,140,82,24,77,7,217,147,120,50,236,166,39,109,179,249,18,88,134,204,44,102,184,242,25,83,141,199,70,12,210,152,115,57,231,173,248,178,108,38,205,135,89,19,146,216,6,76,167,237,51,121,0,75,150,221,49,122,167,236,98,41,244,191,83,24,197,142,196,143,82,25,245,190,99,40,166,237,48,123,151,220,1,74,149,222,3,72,164,239,50,121,247,188,97,42,198,141,80,27,81,26,199,140,96,43,246,189,51,120,165,238,2,73,148,223,55,124,161,234,6,77,144,219,85,30,195,136,100,47,242,185,243,184,101,46,194,137,84,31,145,218,7,76,160,235,54,125,162,233,52,127,147,216,5,78,192,139,86,29,241,186,103,44,102,45,240,187,87,28,193,138,4,79,146,217,53,126,163,232,110,37,248,179,95,20,201,130,12,71,154,209,61,118,171,224,170,225,60,119,155,208,13,70,200,131,94,21,249,178,111,36,251,176,109,38,202,129,92,23,153,210,15,68,168,227,62,117,63,116,169,226,14,69,152,211,93,22,203,128,108,39,250,177,89,18,207,132,104,35,254,181,59,112,173,230,10,65,156,215,157,214,11,64,172,231,58,113,255,180,105,34,206,133,88,19,204,135,90,17,253,182,107,32,174,229,56,115,159,212,9,66,8,67,158,213,57,114,175,228,106,33,252,183,91,16,205,134,0,76,152,212,45,97,181,249,90,22,194,142,119,59,239,163,180,248,44,96,153,213,1,77,238,162,118,58,195,143,91,23,117,57,237,161,88,20,192,140,47,99,183,251,2,78,154,214,193,141,89,21,236,160,116,56,155,215,3,79,182,250,46,98,234,166,114,62,199,139,95,19,176,252,40,100,157,209,5,73,94,18,198,138,115,63,235,167,4,72,156,208,41,101,177,253,159,211,7,75,178,254,42,102,197,137,93,17,232,164,112,60,43,103,179,255,6,74,158,210,113,61,233,165,92,16,196,136,201,133,81,29,228,168,124,48,147,223,11,71,190,242,38,106,125,49,229,169,80,28,200,132,39,107,191,243,10,70,146,222,188,240,36,104,145,221,9,69,230,170,126,50,203,135,83,31,8,68,144,220,37,105,189,241,82,30,202,134,127,51,231,171,35,111,187,247,14,66,150,218,121,53,225,173,84,24,204,128,151,219,15,67,186,246,34,110,205,129,85,25,224,172,120,52,86,26,206,130,123,55,227,175,12,64,148,216,33,109,185,245,226,174,122,54,207,131,87,27,184,244,32,108,149,217,13,65,0,77,154,215,41,100,179,254,82,31,200,133,123,54,225,172,164,233,62,115,141,192,23,90,246,187,108,33,223,146,69,8,85,24,207,130,124,49,230,171,7,74,157,208,46,99,180,249,241,188,107,38,216,149,66,15,163,238,57,116,138,199,16,93,170,231,48,125,131,206,25,84,248,181,98,47,209,156,75,6,14,67,148,217,39,106,189,240,92,17,198,139,117,56,239,162,255,178,101,40,214,155,76,1,173,224,55,122,132,201,30,83,91,22,193,140,114,63,232,165,9,68,147,222,32,109,186,247,73,4,211,158,96,45,250,183,27,86,129,204,50,127,168,229,237,160,119,58,196,137,94,19,191,242,37,104,150,219,12,65,28,81,134,203,53,120,175,226,78,3,212,153,103,42,253,176,184,245,34,111,145,220,11,70,234,167,112,61,195,142,89,20,227,174,121,52,202,135,80,29,177,252,43,102,152,213,2,79,71,10,221,144,110,35,244,185,21,88,143,194,60,113,166,235,182,251,44,97,159,210,5,72,228,169,126,51,205,128,87,26,18,95,136,197,59,118,161,236,64,13,218,151,105,36,243,190,0,78,156,210,37,107,185,247,74,4,214,152,111,33,243,189,148,218,8,70,177,255,45,99,222,144,66,12,251,181,103,41,53,123,169,231,16,94,140,194,127,49,227,173,90,20,198,136,161,239,61,115,132,202,24,86,235,165,119,57,206,128,82,28,106,36,246,184,79,1,211,157,32,110,188,242,5,75,153,215,254,176,98,44,219,149,71,9,180,250,40,102,145,223,13,67,95,17,195,141,122,52,230,168,21,91,137,199,48,126,172,226,203,133,87,25,238,160,114,60,129,207,29,83,164,234,56,118,212,154,72,6,241,191,109,35,158,208,2,76,187,245,39,105,64,14,220,146,101,43,249,183,10,68,150,216,47,97,179,253,225,175,125,51,196,138,88,22,171,229,55,121,142,192,18,92,117,59,233,167,80,30,204,130,63,113,163,237,26,84,134,200,190,240,34,108,155,213,7,73,244,186,104,38,209,159,77,3,42,100,182,248,15,65,147,221,96,46,252,178,69,11,217,151,139,197,23,89,174,224,50,124,193,143,93,19,228,170,120,54,31,81,131,205,58,116,166,232,85,27,201,135,112,62,236,162,0,79,158,209,33,110,191,240,66,13,220,147,99,44,253,178,132,203,26,85,165,234,59,116,198,137,88,23,231,168,121,54,21,90,139,196,52,123,170,229,87,24,201,134,118,57,232,167,145,222,15,64,176,255,46,97,211,156,77,2,242,189,108,35,42,101,180,251,11,68,149,218,104,39,246,185,73,6,215,152,174,225,48,127,143,192,17,94,236,163,114,61,205,130,83,28,63,112,161,238,30,81,128,207,125,50,227,172,92,19,194,141,187,244,37,106,154,213,4,75,249,182,103,40,216,151,70,9,84,27,202,133,117,58,235,164,22,89,136,199,55,120,169,230,208,159,78,1,241,190,111,32,146,221,12,67,179,252,45,98,65,14,223,144,96,47,254,177,3,76,157,210,34,109,188,243,197,138,91,20,228,171,122,53,135,200,25,86,166,233,56,119,126,49,224,175,95,16,193,142,60,115,162,237,29,82,131,204,250,181,100,43,219,148,69,10,184,247,38,105,153,214,7,72,107,36,245,186,74,5,212,155,41,102,183,248,8,71,150,217,239,160,113,62,206,129,80,31,173,226,51,124,140,195,18,93,0,80,160,240,93,13,253,173,186,234,26,74,231,183,71,23,105,57,201,153,52,100,148,196,211,131,115,35,142,222,46,126,210,130,114,34,143,223,47,127,104,56,200,152,53,101,149,197,187,235,27,75,230,182,70,22,1,81,161,241,92,12,252,172,185,233,25,73,228,180,68,20,3,83,163,243,94,14,254,174,208,128,112,32,141,221,45,125,106,58,202,154,55,103,151,199,107,59,203,155,54,102,150,198,209,129,113,33,140,220,44,124,2,82,162,242,95,15,255,175,184,232,24,72,229,181,69,21,111,63,207,159,50,98,146,194,213,133,117,37,136,216,40,120,6,86,166,246,91,11,251,171,188,236,28,76,225,177,65,17,189,237,29,77,224,176,64,16,7,87,167,247,90,10,250,170,212,132,116,36,137,217,41,121,110,62,206,158,51,99,147,195,214,134,118,38,139,219,43,123,108,60,204,156,49,97,145,193,191,239,31,79,226,178,66,18,5,85,165,245,88,8,248,168,4,84,164,244,89,9,249,169,190,238,30,78,227,179,67,19,109,61,205,157,48,96,144,192,215,135,119,39,138,218,42,122,0,81,162,243,89,8,251,170,178,227,16,65,235,186,73,24,121,40,219,138,32,113,130,211,203,154,105,56,146,195,48,97,242,163,80,1,171,250,9,88,64,17,226,179,25,72,187,234,139,218,41,120,210,131,112,33,57,104,155,202,96,49,194,147,249,168,91,10,160,241,2,83,75,26,233,184,18,67,176,225,128,209,34,115,217,136,123,42,50,99,144,193,107,58,201,152,11,90,169,248,82,3,240,161,185,232,27,74,224,177,66,19,114,35,208,129,43,122,137,216,192,145,98,51,153,200,59,106,239,190,77,28,182,231,20,69,93,12,255,174,4,85,166,247,150,199,52,101,207,158,109,60,36,117,134,215,125,44,223,142,29,76,191,238,68,21,230,183,175,254,13,92,246,167,84,5,100,53,198,151,61,108,159,206,214,135,116,37,143,222,45,124,22,71,180,229,79,30,237,188,164,245,6,87,253,172,95,14,111,62,205,156,54,103,148,197,221,140,127,46,132,213,38,119,228,181,70,23,189,236,31,78,86,7,244,165,15,94,173,252,157,204,63,110,196,149,102,55,47,126,141,220,118,39,212,133,0,82,164,246,85,7,241,163,170,248,14,92,255,173,91,9,73,27,237,191,28,78,184,234,227,177,71,21,182,228,18,64,146,192,54,100,199,149,99,49,56,106,156,206,109,63,201,155,219,137,127,45,142,220,42,120,113,35,213,135,36,118,128,210,57,107,157,207,108,62,200,154,147,193,55,101,198,148,98,48,112,34,212,134,37,119,129,211,218,136,126,44,143,221,43,121,171,249,15,93,254,172,90,8,1,83,165,247,84,6,240,162,226,176,70,20,183,229,19,65,72,26,236,190,29,79,185,235,114,32,214,132,39,117,131,209,216,138,124,46,141,223,41,123,59,105,159,205,110,60,202,152,145,195,53,103,196,150,96,50,224,178,68,22,181,231,17,67,74,24,238,188,31,77,187,233,169,251,13,95,252,174,88,10,3,81,167,245,86,4,242,160,75,25,239,189,30,76,186,232,225,179,69,23,180,230,16,66,2,80,166,244,87,5,243,161,168,250,12,94,253,175,89,11,217,139,125,47,140,222,40,122,115,33,215,133,38,116,130,208,144,194,52,102,197,151,97,51,58,104,158,204,111,61,203,153,0,83,166,245,81,2,247,164,162,241,4,87,243,160,85,6,89,10,255,172,8,91,174,253,251,168,93,14,170,249,12,95,178,225,20,71,227,176,69,22,16,67,182,229,65,18,231,180,235,184,77,30,186,233,28,79,73,26,239,188,24,75,190,237,121,42,223,140,40,123,142,221,219,136,125,46,138,217,44,127,32,115,134,213,113,34,215,132,130,209,36,119,211,128,117,38,203,152,109,62,154,201,60,111,105,58,207,156,56,107,158,205,146,193,52,103,195,144,101,54,48,99,150,197,97,50,199,148,242,161,84,7,163,240,5,86,80,3,246,165,1,82,167,244,171,248,13,94,250,169,92,15,9,90,175,252,88,11,254,173,64,19,230,181,17,66,183,228,226,177,68,23,179,224,21,70,25,74,191,236,72,27,238,189,187,232,29,78,234,185,76,31,139,216,45,126,218,137,124,47,41,122,143,220,120,43,222,141,210,129,116,39,131,208,37,118,112,35,214,133,33,114,135,212,57,106,159,204,104,59,206,157,155,200,61,110,202,153,108,63,96,51,198,149,49,98,151,196,194,145,100,55,147,192,53,102,0,84,168,252,77,25,229,177,154,206,50,102,215,131,127,43,41,125,129,213,100,48,204,152,179,231,27,79,254,170,86,2,82,6,250,174,31,75,183,227,200,156,96,52,133,209,45,121,123,47,211,135,54,98,158,202,225,181,73,29,172,248,4,80,164,240,12,88,233,189,65,21,62,106,150,194,115,39,219,143,141,217,37,113,192,148,104,60,23,67,191,235,90,14,242,166,246,162,94,10,187,239,19,71,108,56,196,144,33,117,137,221,223,139,119,35,146,198,58,110,69,17,237,185,8,92,160,244,85,1,253,169,24,76,176,228,207,155,103,51,130,214,42,126,124,40,212,128,49,101,153,205,230,178,78,26,171,255,3,87,7,83,175,251,74,30,226,182,157,201,53,97,208,132,120,44,46,122,134,210,99,55,203,159,180,224,28,72,249,173,81,5,241,165,89,13,188,232,20,64,107,63,195,151,38,114,142,218,216,140,112,36,149,193,61,105,66,22,234,190,15,91,167,243,163,247,11,95,238,186,70,18,57,109,145,197,116,32,220,136,138,222,34,118,199,147,111,59,16,68,184,236,93,9,245,161,0,85,170,255,73,28,227,182,146,199,56,109,219,142,113,36,57,108,147,198,112,37,218,143,171,254,1,84,226,183,72,29,114,39,216,141,59,110,145,196,224,181,74,31,169,252,3,86,75,30,225,180,2,87,168,253,217,140,115,38,144,197,58,111,228,177,78,27,173,248,7,82,118,35,220,137,63,106,149,192,221,136,119,34,148,193,62,107,79,26,229,176,6,83,172,249,150,195,60,105,223,138,117,32,4,81,174,251,77,24,231,178,175,250,5,80,230,179,76,25,61,104,151,194,116,33,222,139,213,128,127,42,156,201,54,99,71,18,237,184,14,91,164,241,236,185,70,19,165,240,15,90,126,43,212,129,55,98,157,200,167,242,13,88,238,187,68,17,53,96,159,202,124,41,214,131,158,203,52,97,215,130,125,40,12,89,166,243,69,16,239,186,49,100,155,206,120,45,210,135,163,246,9,92,234,191,64,21,8,93,162,247,65,20,235,190,154,207,48,101,211,134,121,44,67,22,233,188,10,95,160,245,209,132,123,46,152,205,50,103,122,47,208,133,51,102,153,204,232,189,66,23,161,244,11,94,0,86,172,250,69,19,233,191,138,220,38,112,207,153,99,53,9,95,165,243,76,26,224,182,131,213,47,121,198,144,106,60,18,68,190,232,87,1,251,173,152,206,52,98,221,139,113,39,27,77,183,225,94,8,242,164,145,199,61,107,212,130,120,46,36,114,136,222,97,55,205,155,174,248,2,84,235,189,71,17,45,123,129,215,104,62,196,146,167,241,11,93,226,180,78,24,54,96,154,204,115,37,223,137,188,234,16,70,249,175,85,3,63,105,147,197,122,44,214,128,181,227,25,79,240,166,92,10,72,30,228,178,13,91,161,247,194,148,110,56,135,209,43,125,65,23,237,187,4,82,168,254,203,157,103,49,142,216,34,116,90,12,246,160,31,73,179,229,208,134,124,42,149,195,57,111,83,5,255,169,22,64,186,236,217,143,117,35,156,202,48,102,108,58,192,150,41,127,133,211,230,176,74,28,163,245,15,89,101,51,201,159,32,118,140,218,239,185,67,21,170,252,6,80,126,40,210,132,59,109,151,193,244,162,88,14,177,231,29,75,119,33,219,141,50,100,158,200,253,171,81,7,184,238,20,66,0,87,174,249,65,22,239,184,130,213,44,123,195,148,109,58,25,78,183,224,88,15,246,161,155,204,53,98,218,141,116,35,50,101,156,203,115,36,221,138,176,231,30,73,241,166,95,8,43,124,133,210,106,61,196,147,169,254,7,80,232,191,70,17,100,51,202,157,37,114,139,220,230,177,72,31,167,240,9,94,125,42,211,132,60,107,146,197,255,168,81,6,190,233,16,71,86,1,248,175,23,64,185,238,212,131,122,45,149,194,59,108,79,24,225,182,14,89,160,247,205,154,99,52,140,219,34,117,200,159,102,49,137,222,39,112,74,29,228,179,11,92,165,242,209,134,127,40,144,199,62,105,83,4,253,170,18,69,188,235,250,173,84,3,187,236,21,66,120,47,214,129,57,110,151,192,227,180,77,26,162,245,12,91,97,54,207,152,32,119,142,217,172,251,2,85,237,186,67,20,46,121,128,215,111,56,193,150,181,226,27,76,244,163,90,13,55,96,153,206,118,33,216,143,158,201,48,103,223,136,113,38,28,75,178,229,93,10,243,164,135,208,41,126,198,145,104,63,5,82,171,252,68,19,234,189,0,88,176,232,125,37,205,149,250,162,74,18,135,223,55,111,233,177,89,1,148,204,36,124,19,75,163,251,110,54,222,134,207,151,127,39,178,234,2,90,53,109,133,221,72,16,248,160,38,126,150,206,91,3,235,179,220,132,108,52,161,249,17,73,131,219,51,107,254,166,78,22,121,33,201,145,4,92,180,236,106,50,218,130,23,79,167,255,144,200,32,120,237,181,93,5,76,20,252,164,49,105,129,217,182,238,6,94,203,147,123,35,165,253,21,77,216,128,104,48,95,7,239,183,34,122,146,202,27,67,171,243,102,62,214,142,225,185,81,9,156,196,44,116,242,170,66,26,143,215,63,103,8,80,184,224,117,45,197,157,212,140,100,60,169,241,25,65,46,118,158,198,83,11,227,187,61,101,141,213,64,24,240,168,199,159,119,47,186,226,10,82,152,192,40,112,229,189,85,13,98,58,210,138,31,71,175,247,113,41,193,153,12,84,188,228,139,211,59,99,246,174,70,30,87,15,231,191,42,114,154,194,173,245,29,69,208,136,96,56,190,230,14,86,195,155,115,43,68,28,244,172,57,97,137,209,0,89,178,235,121,32,203,146,242,171,64,25,139,210,57,96,249,160,75,18,128,217,50,107,11,82,185,224,114,43,192,153,239,182,93,4,150,207,36,125,29,68,175,246,100,61,214,143,22,79,164,253,111,54,221,132,228,189,86,15,157,196,47,118,195,154,113,40,186,227,8,81,49,104,131,218,72,17,250,163,58,99,136,209,67,26,241,168,200,145,122,35,177,232,3,90,44,117,158,199,85,12,231,190,222,135,108,53,167,254,21,76,213,140,103,62,172,245,30,71,39,126,149,204,94,7,236,181,155,194,41,112,226,187,80,9,105,48,219,130,16,73,162,251,98,59,208,137,27,66,169,240,144,201,34,123,233,176,91,2,116,45,198,159,13,84,191,230,134,223,52,109,255,166,77,20,141,212,63,102,244,173,70,31,127,38,205,148,6,95,180,237,88,1,234,179,33,120,147,202,170,243,24,65,211,138,97,56,161,248,19,74,216,129,106,51,83,10,225,184,42,115,152,193,183,238,5,92,206,151,124,37,69,28,247,174,60,101,142,215,78,23,252,165,55,110,133,220,188,229,14,87,197,156,119,46,0,90,180,238,117,47,193,155,234,176,94,4,159,197,43,113,201,147,125,39,188,230,8,82,35,121,151,205,86,12,226,184,143,213,59,97,250,160,78,20,101,63,209,139,16,74,164,254,70,28,242,168,51,105,135,221,172,246,24,66,217,131,109,55,3,89,183,237,118,44,194,152,233,179,93,7,156,198,40,114,202,144,126,36,191,229,11,81,32,122,148,206,85,15,225,187,140,214,56,98,249,163,77,23,102,60,210,136,19,73,167,253,69,31,241,171,48,106,132,222,175,245,27,65,218,128,110,52,6,92,178,232,115,41,199,157,236,182,88,2,153,195,45,119,207,149,123,33,186,224,14,84,37,127,145,203,80,10,228,190,137,211,61,103,252,166,72,18,99,57,215,141,22,76,162,248,64,26,244,174,53,111,129,219,170,240,30,68,223,133,107,49,5,95,177,235,112,42,196,158,239,181,91,1,154,192,46,116,204,150,120,34,185,227,13,87,38,124,146,200,83,9,231,189,138,208,62,100,255,165,75,17,96,58,212,142,21,79,161,251,67,25,247,173,54,108,130,216,169,243,29,71,220,134,104,50,0,91,182,237,113,42,199,156,226,185,84,15,147,200,37,126,217,130,111,52,168,243,30,69,59,96,141,214,74,17,252,167,175,244,25,66,222,133,104,51,77,22,251,160,60,103,138,209,118,45,192,155,7,92,177,234,148,207,34,121,229,190,83,8,67,24,245,174,50,105,132,223,161,250,23,76,208,139,102,61,154,193,44,119,235,176,93,6,120,35,206,149,9,82,191,228,236,183,90,1,157,198,43,112,14,85,184,227,127,36,201,146,53,110,131,216,68,31,242,169,215,140,97,58,166,253,16,75,134,221,48,107,247,172,65,26,100,63,210,137,21,78,163,248,95,4,233,178,46,117,152,195,189,230,11,80,204,151,122,33,41,114,159,196,88,3,238,181,203,144,125,38,186,225,12,87,240,171,70,29,129,218,55,108,18,73,164,255,99,56,213,142,197,158,115,40,180,239,2,89,39,124,145,202,86,13,224,187,28,71,170,241,109,54,219,128,254,165,72,19,143,212,57,98,106,49,220,135,27,64,173,246,136,211,62,101,249,162,79,20,179,232,5,94,194,153,116,47,81,10,231,188,32,123,150,205,0,92,184,228,109,49,213,137,218,134,98,62,183,235,15,83,169,245,17,77,196,152,124,32,115,47,203,151,30,66,166,250,79,19,247,171,34,126,154,198,149,201,45,113,248,164,64,28,230,186,94,2,139,215,51,111,60,96,132,216,81,13,233,181,158,194,38,122,243,175,75,23,68,24,252,160,41,117,145,205,55,107,143,211,90,6,226,190,237,177,85,9,128,220,56,100,209,141,105,53,188,224,4,88,11,87,179,239,102,58,222,130,120,36,192,156,21,73,173,241,162,254,26,70,207,147,119,43,33,125,153,197,76,16,244,168,251,167,67,31,150,202,46,114,136,212,48,108,229,185,93,1,82,14,234,182,63,99,135,219,110,50,214,138,3,95,187,231,180,232,12,80,217,133,97,61,199,155,127,35,170,246,18,78,29,65,165,249,112,44,200,148,191,227,7,91,210,142,106,54,101,57,221,129,8,84,176,236,22,74,174,242,123,39,195,159,204,144,116,40,161,253,25,69,240,172,72,20,157,193,37,121,42,118,146,206,71,27,255,163,89,5,225,189,52,104,140,208,131,223,59,103,238,178,86,10,0,93,186,231,105,52,211,142,210,143,104,53,187,230,1,92,185,228,3,94,208,141,106,55,107,54,209,140,2,95,184,229,111,50,213,136,6,91,188,225,189,224,7,90,212,137,110,51,214,139,108,49,191,226,5,88,4,89,190,227,109,48,215,138,222,131,100,57,183,234,13,80,12,81,182,235,101,56,223,130,103,58,221,128,14,83,180,233,181,232,15,82,220,129,102,59,177,236,11,86,216,133,98,63,99,62,217,132,10,87,176,237,8,85,178,239,97,60,219,134,218,135,96,61,179,238,9,84,161,252,27,70,200,149,114,47,115,46,201,148,26,71,160,253,24,69,162,255,113,44,203,150,202,151,112,45,163,254,25,68,206,147,116,41,167,250,29,64,28,65,166,251,117,40,207,146,119,42,205,144,30,67,164,249,165,248,31,66,204,145,118,43,127,34,197,152,22,75,172,241,173,240,23,74,196,153,126,35,198,155,124,33,175,242,21,72,20,73,174,243,125,32,199,154,16,77,170,247,121,36,195,158,194,159,120,37,171,246,17,76,169,244,19,78,192,157,122,39,123,38,193,156,18,79,168,245,0,94,188,226,101,59,217,135,202,148,118,40,175,241,19,77,137,215,53,107,236,178,80,14,67,29,255,161,38,120,154,196,15,81,179,237,106,52,214,136,197,155,121,39,160,254,28,66,134,216,58,100,227,189,95,1,76,18,240,174,41,119,149,203,30,64,162,252,123,37,199,153,212,138,104,54,177,239,13,83,151,201,43,117,242,172,78,16,93,3,225,191,56,102,132,218,17,79,173,243,116,42,200,150,219,133,103,57,190,224,2,92,152,198,36,122,253,163,65,31,82,12,238,176,55,105,139,213,60,98,128,222,89,7,229,187,246,168,74,20,147,205,47,113,181,235,9,87,208,142,108,50,127,33,195,157,26,68,166,248,51,109,143,209,86,8,234,180,249,167,69,27,156,194,32,126,186,228,6,88,223,129,99,61,112,46,204,146,21,75,169,247,34,124,158,192,71,25,251,165,232,182,84,10,141,211,49,111,171,245,23,73,206,144,114,44,97,63,221,131,4,90,184,230,45,115,145,207,72,22,244,170,231,185,91,5,130,220,62,96,164,250,24,70,193,159,125,35,110,48,210,140,11,85,183,233,0,95,190,225,97,62,223,128,194,157,124,35,163,252,29,66,153,198,39,120,248,167,70,25,91,4,229,186,58,101,132,219,47,112,145,206,78,17,240,175,237,178,83,12,140,211,50,109,182,233,8,87,215,136,105,54,116,43,202,149,21,74,171,244,94,1,224,191,63,96,129,222,156,195,34,125,253,162,67,28,199,152,121,38,166,249,24,71,5,90,187,228,100,59,218,133,113,46,207,144,16,79,174,241,179,236,13,82,210,141,108,51,232,183,86,9,137,214,55,104,42,117,148,203,75,20,245,170,188,227,2,93,221,130,99,60,126,33,192,159,31,64,161,254,37,122,155,196,68,27,250,165,231,184,89,6,134,217,56,103,147,204,45,114,242,173,76,19,81,14,239,176,48,111,142,209,10,85,180,235,107,52,213,138,200,151,118,41,169,246,23,72,226,189,92,3,131,220,61,98,32,127,158,193,65,30,255,160,123,36,197,154,26,69,164,251,185,230,7,88,216,135,102,57,205,146,115,44,172,243,18,77,15,80,177,238,110,49,208,143,84,11,234,181,53,106,139,212,150,201,40,119,247,168,73,22,0,96,192,160,157,253,93,61,39,71,231,135,186,218,122,26,78,46,142,238,211,179,19,115,105,9,169,201,244,148,52,84,156,252,92,60,1,97,193,161,187,219,123,27,38,70,230,134,210,178,18,114,79,47,143,239,245,149,53,85,104,8,168,200,37,69,229,133,184,216,120,24,2,98,194,162,159,255,95,63,107,11,171,203,246,150,54,86,76,44,140,236,209,177,17,113,185,217,121,25,36,68,228,132,158,254,94,62,3,99,195,163,247,151,55,87,106,10,170,202,208,176,16,112,77,45,141,237,74,42,138,234,215,183,23,119,109,13,173,205,240,144,48,80,4,100,196,164,153,249,89,57,35,67,227,131,190,222,126,30,214,182,22,118,75,43,139,235,241,145,49,81,108,12,172,204,152,248,88,56,5,101,197,165,191,223,127,31,34,66,226,130,111,15,175,207,242,146,50,82,72,40,136,232,213,181,21,117,33,65,225,129,188,220,124,28,6,102,198,166,155,251,91,59,243,147,51,83,110,14,174,206,212,180,20,116,73,41,137,233,189,221,125,29,32,64,224,128,154,250,90,58,7,103,199,167,0,97,194,163,153,248,91,58,47,78,237,140,182,215,116,21,94,63,156,253,199,166,5,100,113,16,179,210,232,137,42,75,188,221,126,31,37,68,231,134,147,242,81,48,10,107,200,169,226,131,32,65,123,26,185,216,205,172,15,110,84,53,150,247,101,4,167,198,252,157,62,95,74,43,136,233,211,178,17,112,59,90,249,152,162,195,96,1,20,117,214,183,141,236,79,46,217,184,27,122,64,33,130,227,246,151,52,85,111,14,173,204,135,230,69,36,30,127,220,189,168,201,106,11,49,80,243,146,202,171,8,105,83,50,145,240,229,132,39,70,124,29,190,223,148,245,86,55,13,108,207,174,187,218,121,24,34,67,224,129,118,23,180,213,239,142,45,76,89,56,155,250,192,161,2,99,40,73,234,139,177,208,115,18,7,102,197,164,158,255,92,61,175,206,109,12,54,87,244,149,128,225,66,35,25,120,219,186,241,144,51,82,104,9,170,203,222,191,28,125,71,38,133,228,19,114,209,176,138,235,72,41,60,93,254,159,165,196,103,6,77,44,143,238,212,181,22,119,98,3,160,193,251,154,57,88,0,98,196,166,149,247,81,51,55,85,243,145,162,192,102,4,110,12,170,200,251,153,63,93,89,59,157,255,204,174,8,106,220,190,24,122,73,43,141,239,235,137,47,77,126,28,186,216,178,208,118,20,39,69,227,129,133,231,65,35,16,114,212,182,165,199,97,3,48,82,244,150,146,240,86,52,7,101,195,161,203,169,15,109,94,60,154,248,252,158,56,90,105,11,173,207,121,27,189,223,236,142,40,74,78,44,138,232,219,185,31,125,23,117,211,177,130,224,70,36,32,66,228,134,181,215,113,19,87,53,147,241,194,160,6,100,96,2,164,198,245,151,49,83,57,91,253,159,172,206,104,10,14,108,202,168,155,249,95,61,139,233,79,45,30,124,218,184,188,222,120,26,41,75,237,143,229,135,33,67,112,18,180,214,210,176,22,116,71,37,131,225,242,144,54,84,103,5,163,193,197,167,1,99,80,50,148,246,156,254,88,58,9,107,205,175,171,201,111,13,62,92,250,152,46,76,234,136,187,217,127,29,25,123,221,191,140,238,72,42,64,34,132,230,213,183,17,115,119,21,179,209,226,128,38,68,0,99,198,165,145,242,87,52,63,92,249,154,174,205,104,11,126,29,184,219,239,140,41,74,65,34,135,228,208,179,22,117,252,159,58,89,109,14,171,200,195,160,5,102,82,49,148,247,130,225,68,39,19,112,213,182,189,222,123,24,44,79,234,137,229,134,35,64,116,23,178,209,218,185,28,127,75,40,141,238,155,248,93,62,10,105,204,175,164,199,98,1,53,86,243,144,25,122,223,188,136,235,78,45,38,69,224,131,183,212,113,18,103,4,161,194,246,149,48,83,88,59,158,253,201,170,15,108,215,180,17,114,70,37,128,227,232,139,46,77,121,26,191,220,169,202,111,12,56,91,254,157,150,245,80,51,7,100,193,162,43,72,237,142,186,217,124,31,20,119,210,177,133,230,67,32,85,54,147,240,196,167,2,97,106,9,172,207,251,152,61,94,50,81,244,151,163,192,101,6,13,110,203,168,156,255,90,57,76,47,138,233,221,190,27,120,115,16,181,214,226,129,36,71,206,173,8,107,95,60,153,250,241,146,55,84,96,3,166,197,176,211,118,21,33,66,231,132,143,236,73,42,30,125,216,187,0,100,200,172,141,233,69,33,7,99,207,171,138,238,66,38,14,106,198,162,131,231,75,47,9,109,193,165,132,224,76,40,28,120,212,176,145,245,89,61,27,127,211,183,150,242,94,58,18,118,218,190,159,251,87,51,21,113,221,185,152,252,80,52,56,92,240,148,181,209,125,25,63,91,247,147,178,214,122,30,54,82,254,154,187,223,115,23,49,85,249,157,188,216,116,16,36,64,236,136,169,205,97,5,35,71,235,143,174,202,102,2,42,78,226,134,167,195,111,11,45,73,229,129,160,196,104,12,112,20,184,220,253,153,53,81,119,19,191,219,250,158,50,86,126,26,182,210,243,151,59,95,121,29,177,213,244,144,60,88,108,8,164,192,225,133,41,77,107,15,163,199,230,130,46,74,98,6,170,206,239,139,39,67,101,1,173,201,232,140,32,68,72,44,128,228,197,161,13,105,79,43,135,227,194,166,10,110,70,34,142,234,203,175,3,103,65,37,137,237,204,168,4,96,84,48,156,248,217,189,17,117,83,55,155,255,222,186,22,114,90,62,146,246,215,179,31,123,93,57,149,241,208,180,24,124,0,101,202,175,137,236,67,38,15,106,197,160,134,227,76,41,30,123,212,177,151,242,93,56,17,116,219,190,152,253,82,55,60,89,246,147,181,208,127,26,51,86,249,156,186,223,112,21,34,71,232,141,171,206,97,4,45,72,231,130,164,193,110,11,120,29,178,215,241,148,59,94,119,18,189,216,254,155,52,81,102,3,172,201,239,138,37,64,105,12,163,198,224,133,42,79,68,33,142,235,205,168,7,98,75,46,129,228,194,167,8,109,90,63,144,245,211,182,25,124,85,48,159,250,220,185,22,115,240,149,58,95,121,28,179,214,255,154,53,80,118,19,188,217,238,139,36,65,103,2,173,200,225,132,43,78,104,13,162,199,204,169,6,99,69,32,143,234,195,166,9,108,74,47,128,229,210,183,24,125,91,62,145,244,221,184,23,114,84,49,158,251,136,237,66,39,1,100,203,174,135,226,77,40,14,107,196,161,150,243,92,57,31,122,213,176,153,252,83,54,16,117,218,191,180,209,126,27,61,88,247,146,187,222,113,20,50,87,248,157,170,207,96,5,35,70,233,140,165,192,111,10,44,73,230,131,0,102,204,170,133,227,73,47,23,113,219,189,146,244,94,56,46,72,226,132,171,205,103,1,57,95,245,147,188,218,112,22,92,58,144,246,217,191,21,115,75,45,135,225,206,168,2,100,114,20,190,216,247,145,59,93,101,3,169,207,224,134,44,74,184,222,116,18,61,91,241,151,175,201,99,5,42,76,230,128,150,240,90,60,19,117,223,185,129,231,77,43,4,98,200,174,228,130,40,78,97,7,173,203,243,149,63,89,118,16,186,220,202,172,6,96,79,41,131,229,221,187,17,119,88,62,148,242,109,11,161,199,232,142,36,66,122,28,182,208,255,153,51,85,67,37,143,233,198,160,10,108,84,50,152,254,209,183,29,123,49,87,253,155,180,210,120,30,38,64,234,140,163,197,111,9,31,121,211,181,154,252,86,48,8,110,196,162,141,235,65,39,213,179,25,127,80,54,156,250,194,164,14,104,71,33,139,237,251,157,55,81,126,24,178,212,236,138,32,70,105,15,165,195,137,239,69,35,12,106,192,166,158,248,82,52,27,125,215,177,167,193,107,13,34,68,238,136,176,214,124,26,53,83,249,159,0,103,206,169,129,230,79,40,31,120,209,182,158,249,80,55,62,89,240,151,191,216,113,22,33,70,239,136,160,199,110,9,124,27,178,213,253,154,51,84,99,4,173,202,226,133,44,75,66,37,140,235,195,164,13,106,93,58,147,244,220,187,18,117,248,159,54,81,121,30,183,208,231,128,41,78,102,1,168,207,198,161,8,111,71,32,137,238,217,190,23,112,88,63,150,241,132,227,74,45,5,98,203,172,155,252,85,50,26,125,212,179,186,221,116,19,59,92,245,146,165,194,107,12,36,67,234,141,237,138,35,68,108,11,162,197,242,149,60,91,115,20,189,218,211,180,29,122,82,53,156,251,204,171,2,101,77,42,131,228,145,246,95,56,16,119,222,185,142,233,64,39,15,104,193,166,175,200,97,6,46,73,224,135,176,215,126,25,49,86,255,152,21,114,219,188,148,243,90,61,10,109,196,163,139,236,69,34,43,76,229,130,170,205,100,3,52,83,250,157,181,210,123,28,105,14,167,192,232,143,38,65,118,17,184,223,247,144,57,94,87,48,153,254,214,177,24,127,72,47,134,225,201,174,7,96,0,104,208,184,189,213,109,5,103,15,183,223,218,178,10,98,206,166,30,118,115,27,163,203,169,193,121,17,20,124,196,172,129,233,81,57,60,84,236,132,230,142,54,94,91,51,139,227,79,39,159,247,242,154,34,74,40,64,248,144,149,253,69,45,31,119,207,167,162,202,114,26,120,16,168,192,197,173,21,125,209,185,1,105,108,4,188,212,182,222,102,14,11,99,219,179,158,246,78,38,35,75,243,155,249,145,41,65,68,44,148,252,80,56,128,232,237,133,61,85,55,95,231,143,138,226,90,50,62,86,238,134,131,235,83,59,89,49,137,225,228,140,52,92,240,152,32,72,77,37,157,245,151,255,71,47,42,66,250,146,191,215,111,7,2,106,210,186,216,176,8,96,101,13,181,221,113,25,161,201,204,164,28,116,22,126,198,174,171,195,123,19,33,73,241,153,156,244,76,36,70,46,150,254,251,147,43,67,239,135,63,87,82,58,130,234,136,224,88,48,53,93,229,141,160,200,112,24,29,117,205,165,199,175,23,127,122,18,170,194,110,6,190,214,211,187,3,107,9,97,217,177,180,220,100,12,0,105,210,187,185,208,107,2,111,6,189,212,214,191,4,109,222,183,12,101,103,14,181,220,177,216,99,10,8,97,218,179,161,200,115,26,24,113,202,163,206,167,28,117,119,30,165,204,127,22,173,196,198,175,20,125,16,121,194,171,169,192,123,18,95,54,141,228,230,143,52,93,48,89,226,139,137,224,91,50,129,232,83,58,56,81,234,131,238,135,60,85,87,62,133,236,254,151,44,69,71,46,149,252,145,248,67,42,40,65,250,147,32,73,242,155,153,240,75,34,79,38,157,244,246,159,36,77,190,215,108,5,7,110,213,188,209,184,3,106,104,1,186,211,96,9,178,219,217,176,11,98,15,102,221,180,182,223,100,13,31,118,205,164,166,207,116,29,112,25,162,203,201,160,27,114,193,168,19,122,120,17,170,195,174,199,124,21,23,126,197,172,225,136,51,90,88,49,138,227,142,231,92,53,55,94,229,140,63,86,237,132,134,239,84,61,80,57,130,235,233,128,59,82,64,41,146,251,249,144,43,66,47,70,253,148,150,255,68,45,158,247,76,37,39,78,245,156,241,152,35,74,72,33,154,243,0,106,212,190,181,223,97,11,119,29,163,201,194,168,22,124,238,132,58,80,91,49,143,229,153,243,77,39,44,70,248,146,193,171,21,127,116,30,160,202,182,220,98,8,3,105,215,189,47,69,251,145,154,240,78,36,88,50,140,230,237,135,57,83,159,245,75,33,42,64,254,148,232,130,60,86,93,55,137,227,113,27,165,207,196,174,16,122,6,108,210,184,179,217,103,13,94,52,138,224,235,129,63,85,41,67,253,151,156,246,72,34,176,218,100,14,5,111,209,187,199,173,19,121,114,24,166,204,35,73,247,157,150,252,66,40,84,62,128,234,225,139,53,95,205,167,25,115,120,18,172,198,186,208,110,4,15,101,219,177,226,136,54,92,87,61,131,233,149,255,65,43,32,74,244,158,12,102,216,178,185,211,109,7,123,17,175,197,206,164,26,112,188,214,104,2,9,99,221,183,203,161,31,117,126,20,170,192,82,56,134,236,231,141,51,89,37,79,241,155,144,250,68,46,125,23,169,195,200,162,28,118,10,96,222,180,191,213,107,1,147,249,71,45,38,76,242,152,228,142,48,90,81,59,133,239,0,107,214,189,177,218,103,12,127,20,169,194,206,165,24,115,254,149,40,67,79,36,153,242,129,234,87,60,48,91,230,141,225,138,55,92,80,59,134,237,158,245,72,35,47,68,249,146,31,116,201,162,174,197,120,19,96,11,182,221,209,186,7,108,223,180,9,98,110,5,184,211,160,203,118,29,17,122,199,172,33,74,247,156,144,251,70,45,94,53,136,227,239,132,57,82,62,85,232,131,143,228,89,50,65,42,151,252,240,155,38,77,192,171,22,125,113,26,167,204,191,212,105,2,14,101,216,179,163,200,117,30,18,121,196,175,220,183,10,97,109,6,187,208,93,54,139,224,236,135,58,81,34,73,244,159,147,248,69,46,66,41,148,255,243,152,37,78,61,86,235,128,140,231,90,49,188,215,106,1,13,102,219,176,195,168,21,126,114,25,164,207,124,23,170,193,205,166,27,112,3,104,213,190,178,217,100,15,130,233,84,63,51,88,229,142,253,150,43,64,76,39,154,241,157,246,75,32,44,71,250,145,226,137,52,95,83,56,133,238,99,8,181,222,210,185,4,111,28,119,202,161,173,198,123,16,0,108,216,180,173,193,117,25,71,43,159,243,234,134,50,94,142,226,86,58,35,79,251,151,201,165,17,125,100,8,188,208,1,109,217,181,172,192,116,24,70,42,158,242,235,135,51,95,143,227,87,59,34,78,250,150,200,164,16,124,101,9,189,209,2,110,218,182,175,195,119,27,69,41,157,241,232,132,48,92,140,224,84,56,33,77,249,149,203,167,19,127,102,10,190,210,3,111,219,183,174,194,118,26,68,40,156,240,233,133,49,93,141,225,85,57,32,76,248,148,202,166,18,126,103,11,191,211,4,104,220,176,169,197,113,29,67,47,155,247,238,130,54,90,138,230,82,62,39,75,255,147,205,161,21,121,96,12,184,212,5,105,221,177,168,196,112,28,66,46,154,246,239,131,55,91,139,231,83,63,38,74,254,146,204,160,20,120,97,13,185,213,6,106,222,178,171,199,115,31,65,45,153,245,236,128,52,88,136,228,80,60,37,73,253,145,207,163,23,123,98,14,186,214,7,107,223,179,170,198,114,30,64,44,152,244,237,129,53,89,137,229,81,61,36,72,252,144,206,162,22,122,99,15,187,215,0,109,218,183,169,196,115,30,79,34,149,248,230,139,60,81,158,243,68,41,55,90,237,128,209,188,11,102,120,21,162,207,33,76,251,150,136,229,82,63,110,3,180,217,199,170,29,112,191,210,101,8,22,123,204,161,240,157,42,71,89,52,131,238,66,47,152,245,235,134,49,92,13,96,215,186,164,201,126,19,220,177,6,107,117,24,175,194,147,254,73,36,58,87,224,141,99,14,185,212,202,167,16,125,44,65,246,155,133,232,95,50,253,144,39,74,84,57,142,227,178,223,104,5,27,118,193,172,132,233,94,51,45,64,247,154,203,166,17,124,98,15,184,213,26,119,192,173,179,222,105,4,85,56,143,226,252,145,38,75,165,200,127,18,12,97,214,187,234,135,48,93,67,46,153,244,59,86,225,140,146,255,72,37,116,25,174,195,221,176,7,106,198,171,28,113,111,2,181,216,137,228,83,62,32,77,250,151,88,53,130,239,241,156,43,70,23,122,205,160,190,211,100,9,231,138,61,80,78,35,148,249,168,197,114,31,1,108,219,182,121,20,163,206,208,189,10,103,54,91,236,129,159,242,69,40,0,110,220,178,165,203,121,23,87,57,139,229,242,156,46,64,174,192,114,28,11,101,215,185,249,151,37,75,92,50,128,238,65,47,157,243,228,138,56,86,22,120,202,164,179,221,111,1,239,129,51,93,74,36,150,248,184,214,100,10,29,115,193,175,130,236,94,48,39,73,251,149,213,187,9,103,112,30,172,194,44,66,240,158,137,231,85,59,123,21,167,201,222,176,2,108,195,173,31,113,102,8,186,212,148,250,72,38,49,95,237,131,109,3,177,223,200,166,20,122,58,84,230,136,159,241,67,45,25,119,197,171,188,210,96,14,78,32,146,252,235,133,55,89,183,217,107,5,18,124,206,160,224,142,60,82,69,43,153,247,88,54,132,234,253,147,33,79,15,97,211,189,170,196,118,24,246,152,42,68,83,61,143,225,161,207,125,19,4,106,216,182,155,245,71,41,62,80,226,140,204,162,16,126,105,7,181,219,53,91,233,135,144,254,76,34,98,12,190,208,199,169,27,117,218,180,6,104,127,17,163,205,141,227,81,63,40,70,244,154,116,26,168,198,209,191,13,99,35,77,255,145,134,232,90,52,0,111,222,177,161,206,127,16,95,48,129,238,254,145,32,79,190,209,96,15,31,112,193,174,225,142,63,80,64,47,158,241,97,14,191,208,192,175,30,113,62,81,224,143,159,240,65,46,223,176,1,110,126,17,160,207,128,239,94,49,33,78,255,144,194,173,28,115,99,12,189,210,157,242,67,44,60,83,226,141,124,19,162,205,221,178,3,108,35,76,253,146,130,237,92,51,163,204,125,18,2,109,220,179,252,147,34,77,93,50,131,236,29,114,195,172,188,211,98,13,66,45,156,243,227,140,61,82,153,246,71,40,56,87,230,137,198,169,24,119,103,8,185,214,39,72,249,150,134,233,88,55,120,23,166,201,217,182,7,104,248,151,38,73,89,54,135,232,167,200,121,22,6,105,216,183,70,41,152,247,231,136,57,86,25,118,199,168,184,215,102,9,91,52,133,234,250,149,36,75,4,107,218,181,165,202,123,20,229,138,59,84,68,43,154,245,186,213,100,11,27,116,197,170,58,85,228,139,155,244,69,42,101,10,187,212,196,171,26,117,132,235,90,53,37,74,251,148,219,180,5,106,122,21,164,203,0,112,224,144,221,173,61,77,167,215,71,55,122,10,154,234,83,35,179,195,142,254,110,30,244,132,20,100,41,89,201,185,166,214,70,54,123,11,155,235,1,113,225,145,220,172,60,76,245,133,21,101,40,88,200,184,82,34,178,194,143,255,111,31,81,33,177,193,140,252,108,28,246,134,22,102,43,91,203,187,2,114,226,146,223,175,63,79,165,213,69,53,120,8,152,232,247,135,23,103,42,90,202,186,80,32,176,192,141,253,109,29,164,212,68,52,121,9,153,233,3,115,227,147,222,174,62,78,162,210,66,50,127,15,159,239,5,117,229,149,216,168,56,72,241,129,17,97,44,92,204,188,86,38,182,198,139,251,107,27,4,116,228,148,217,169,57,73,163,211,67,51,126,14,158,238,87,39,183,199,138,250,106,26,240,128,16,96,45,93,205,189,243,131,19,99,46,94,206,190,84,36,180,196,137,249,105,25,160,208,64,48,125,13,157,237,7,119,231,151,218,170,58,74,85,37,181,197,136,248,104,24,242,130,18,98,47,95,207,191,6,118,230,150,219,171,59,75,161,209,65,49,124,12,156,236,0,113,226,147,217,168,59,74,175,222,77,60,118,7,148,229,67,50,161,208,154,235,120,9,236,157,14,127,53,68,215,166,134,247,100,21,95,46,189,204,41,88,203,186,240,129,18,99,197,180,39,86,28,109,254,143,106,27,136,249,179,194,81,32,17,96,243,130,200,185,42,91,190,207,92,45,103,22,133,244,82,35,176,193,139,250,105,24,253,140,31,110,36,85,198,183,151,230,117,4,78,63,172,221,56,73,218,171,225,144,3,114,212,165,54,71,13,124,239,158,123,10,153,232,162,211,64,49,34,83,192,177,251,138,25,104,141,252,111,30,84,37,182,199,97,16,131,242,184,201,90,43,206,191,44,93,23,102,245,132,164,213,70,55,125,12,159,238,11,122,233,152,210,163,48,65,231,150,5,116,62,79,220,173,72,57,170,219,145,224,115,2,51,66,209,160,234,155,8,121,156,237,126,15,69,52,167,214,112,1,146,227,169,216,75,58,223,174,61,76,6,119,228,149,181,196,87,38,108,29,142,255,26,107,248,137,195,178,33,80,246,135,20,101,47,94,205,188,89,40,187,202,128,241,98,19,0,114,228,150,213,167,49,67,183,197,83,33,98,16,134,244,115,1,151,229,166,212,66,48,196,182,32,82,17,99,245,135,230,148,2,112,51,65,215,165,81,35,181,199,132,246,96,18,149,231,113,3,64,50,164,214,34,80,198,180,247,133,19,97,209,163,53,71,4,118,224,146,102,20,130,240,179,193,87,37,162,208,70,52,119,5,147,225,21,103,241,131,192,178,36,86,55,69,211,161,226,144,6,116,128,242,100,22,85,39,177,195,68,54,160,210,145,227,117,7,243,129,23,101,38,84,194,176,191,205,91,41,106,24,142,252,8,122,236,158,221,175,57,75,204,190,40,90,25,107,253,143,123,9,159,237,174,220,74,56,89,43,189,207,140,254,104,26,238,156,10,120,59,73,223,173,42,88,206,188,255,141,27,105,157,239,121,11,72,58,172,222,110,28,138,248,187,201,95,45,217,171,61,79,12,126,232,154,29,111,249,139,200,186,44,94,170,216,78,60,127,13,155,233,136,250,108,30,93,47,185,203,63,77,219,169,234,152,14,124,251,137,31,109,46,92,202,184,76,62,168,218,153,235,125,15,0,115,230,149,209,162,55,68,191,204,89,42,110,29,136,251,99,16,133,246,178,193,84,39,220,175,58,73,13,126,235,152,198,181,32,83,23,100,241,130,121,10,159,236,168,219,78,61,165,214,67,48,116,7,146,225,26,105,252,143,203,184,45,94,145,226,119,4,64,51,166,213,46,93,200,187,255,140,25,106,242,129,20,103,35,80,197,182,77,62,171,216,156,239,122,9,87,36,177,194,134,245,96,19,232,155,14,125,57,74,223,172,52,71,210,161,229,150,3,112,139,248,109,30,90,41,188,207,63,76,217,170,238,157,8,123,128,243,102,21,81,34,183,196,92,47,186,201,141,254,107,24,227,144,5,118,50,65,212,167,249,138,31,108,40,91,206,189,70,53,160,211,151,228,113,2,154,233,124,15,75,56,173,222,37,86,195,176,244,135,18,97,174,221,72,59,127,12,153,234,17,98,247,132,192,179,38,85,205,190,43,88,28,111,250,137,114,1,148,231,163,208,69,54,104,27,142,253,185,202,95,44,215,164,49,66,6,117,224,147,11,120,237,158,218,169,60,79,180,199,82,33,101,22,131,240,0,116,232,156,205,185,37,81,135,243,111,27,74,62,162,214,19,103,251,143,222,170,54,66,148,224,124,8,89,45,177,197,38,82,206,186,235,159,3,119,161,213,73,61,108,24,132,240,53,65,221,169,248,140,16,100,178,198,90,46,127,11,151,227,76,56,164,208,129,245,105,29,203,191,35,87,6,114,238,154,95,43,183,195,146,230,122,14,216,172,48,68,21,97,253,137,106,30,130,246,167,211,79,59,237,153,5,113,32,84,200,188,121,13,145,229,180,192,92,40,254,138,22,98,51,71,219,175,152,236,112,4,85,33,189,201,31,107,247,131,210,166,58,78,139,255,99,23,70,50,174,218,12,120,228,144,193,181,41,93,190,202,86,34,115,7,155,239,57,77,209,165,244,128,28,104,173,217,69,49,96,20,136,252,42,94,194,182,231,147,15,123,212,160,60,72,25,109,241,133,83,39,187,207,158,234,118,2,199,179,47,91,10,126,226,150,64,52,168,220,141,249,101,17,242,134,26,110,63,75,215,163,117,1,157,233,184,204,80,36,225,149,9,125,44,88,196,176,102,18,142,250,171,223,67,55,0,117,234,159,201,188,35,86,143,250,101,16,70,51,172,217,3,118,233,156,202,191,32,85,140,249,102,19,69,48,175,218,6,115,236,153,207,186,37,80,137,252,99,22,64,53,170,223,5,112,239,154,204,185,38,83,138,255,96,21,67,54,169,220,12,121,230,147,197,176,47,90,131,246,105,28,74,63,160,213,15,122,229,144,198,179,44,89,128,245,106,31,73,60,163,214,10,127,224,149,195,182,41,92,133,240,111,26,76,57,166,211,9,124,227,150,192,181,42,95,134,243,108,25,79,58,165,208,24,109,242,135,209,164,59,78,151,226,125,8,94,43,180,193,27,110,241,132,210,167,56,77,148,225,126,11,93,40,183,194,30,107,244,129,215,162,61,72,145,228,123,14,88,45,178,199,29,104,247,130,212,161,62,75,146,231,120,13,91,46,177,196,20,97,254,139,221,168,55,66,155,238,113,4,82,39,184,205,23,98,253,136,222,171,52,65,152,237,114,7,81,36,187,206,18,103,248,141,219,174,49,68,157,232,119,2,84,33,190,203,17,100,251,142,216,173,50,71,158,235,116,1,87,34,189,200,0,118,236,154,197,179,41,95,151,225,123,13,82,36,190,200,51,69,223,169,246,128,26,108,164,210,72,62,97,23,141,251,102,16,138,252,163,213,79,57,241,135,29,107,52,66,216,174,85,35,185,207,144,230,124,10,194,180,46,88,7,113,235,157,204,186,32,86,9,127,229,147,91,45,183,193,158,232,114,4,255,137,19,101,58,76,214,160,104,30,132,242,173,219,65,55,170,220,70,48,111,25,131,245,61,75,209,167,248,142,20,98,153,239,117,3,92,42,176,198,14,120,226,148,203,189,39,81,133,243,105,31,64,54,172,218,18,100,254,136,215,161,59,77,182,192,90,44,115,5,159,233,33,87,205,187,228,146,8,126,227,149,15,121,38,80,202,188,116,2,152,238,177,199,93,43,208,166,60,74,21,99,249,143,71,49,171,221,130,244,110,24,73,63,165,211,140,250,96,22,222,168,50,68,27,109,247,129,122,12,150,224,191,201,83,37,237,155,1,119,40,94,196,178,47,89,195,181,234,156,6,112,184,206,84,34,125,11,145,231,28,106,240,134,217,175,53,67,139,253,103,17,78,56,162,212,0,119,238,153,193,182,47,88,159,232,113,6,94,41,176,199,35,84,205,186,226,149,12,123,188,203,82,37,125,10,147,228,70,49,168,223,135,240,105,30,217,174,55,64,24,111,246,129,101,18,139,252,164,211,74,61,250,141,20,99,59,76,213,162,140,251,98,21,77,58,163,212,19,100,253,138,210,165,60,75,175,216,65,54,110,25,128,247,48,71,222,169,241,134,31,104,202,189,36,83,11,124,229,146,85,34,187,204,148,227,122,13,233,158,7,112,40,95,198,177,118,1,152,239,183,192,89,46,5,114,235,156,196,179,42,93,154,237,116,3,91,44,181,194,38,81,200,191,231,144,9,126,185,206,87,32,120,15,150,225,67,52,173,218,130,245,108,27,220,171,50,69,29,106,243,132,96,23,142,249,161,214,79,56,255,136,17,102,62,73,208,167,137,254,103,16,72,63,166,209,22,97,248,143,215,160,57,78,170,221,68,51,107,28,133,242,53,66,219,172,244,131,26,109,207,184,33,86,14,121,224,151,80,39,190,201,145,230,127,8,236,155,2,117,45,90,195,180,115,4,157,234,178,197,92,43,0,120,240,136,253,133,13,117,231,159,23,111,26,98,234,146,211,171,35,91,46,86,222,166,52,76,196,188,201,177,57,65,187,195,75,51,70,62,182,206,92,36,172,212,161,217,81,41,104,16,152,224,149,237,101,29,143,247,127,7,114,10,130,250,107,19,155,227,150,238,102,30,140,244,124,4,113,9,129,249,184,192,72,48,69,61,181,205,95,39,175,215,162,218,82,42,208,168,32,88,45,85,221,165,55,79,199,191,202,178,58,66,3,123,243,139,254,134,14,118,228,156,20,108,25,97,233,145,214,174,38,94,43,83,219,163,49,73,193,185,204,180,60,68,5,125,245,141,248,128,8,112,226,154,18,106,31,103,239,151,109,21,157,229,144,232,96,24,138,242,122,2,119,15,135,255,190,198,78,54,67,59,179,203,89,33,169,209,164,220,84,44,189,197,77,53,64,56,176,200,90,34,170,210,167,223,87,47,110,22,158,230,147,235,99,27,137,241,121,1,116,12,132,252,6,126,246,142,251,131,11,115,225,153,17,105,28,100,236,148,213,173,37,93,40,80,216,160,50,74,194,186,207,183,63,71,0,121,242,139,249,128,11,114,239,150,29,100,22,111,228,157,195,186,49,72,58,67,200,177,44,85,222,167,213,172,39,94,155,226,105,16,98,27,144,233,116,13,134,255,141,244,127,6,88,33,170,211,161,216,83,42,183,206,69,60,78,55,188,197,43,82,217,160,210,171,32,89,196,189,54,79,61,68,207,182,232,145,26,99,17,104,227,154,7,126,245,140,254,135,12,117,176,201,66,59,73,48,187,194,95,38,173,212,166,223,84,45,115,10,129,248,138,243,120,1,156,229,110,23,101,28,151,238,86,47,164,221,175,214,93,36,185,192,75,50,64,57,178,203,149,236,103,30,108,21,158,231,122,3,136,241,131,250,113,8,205,180,63,70,52,77,198,191,34,91,208,169,219,162,41,80,14,119,252,133,247,142,5,124,225,152,19,106,24,97,234,147,125,4,143,246,132,253,118,15,146,235,96,25,107,18,153,224,190,199,76,53,71,62,181,204,81,40,163,218,168,209,90,35,230,159,20,109,31,102,237,148,9,112,251,130,240,137,2,123,37,92,215,174,220,165,46,87,202,179,56,65,51,74,193,184,0,122,244,142,245,143,1,123,247,141,3,121,2,120,246,140,243,137,7,125,6,124,242,136,4,126,240,138,241,139,5,127,251,129,15,117,14,116,250,128,12,118,248,130,249,131,13,119,8,114,252,134,253,135,9,115,255,133,11,113,10,112,254,132,235,145,31,101,30,100,234,144,28,102,232,146,233,147,29,103,24,98,236,150,237,151,25,99,239,149,27,97,26,96,238,148,16,106,228,158,229,159,17,107,231,157,19,105,18,104,230,156,227,153,23,109,22,108,226,152,20,110,224,154,225,155,21,111,203,177,63,69,62,68,202,176,60,70,200,178,201,179,61,71,56,66,204,182,205,183,57,67,207,181,59,65,58,64,206,180,48,74,196,190,197,191,49,75,199,189,51,73,50,72,198,188,195,185,55,77,54,76,194,184,52,78,192,186,193,187,53,79,32,90,212,174,213,175,33,91,215,173,35,89,34,88,214,172,211,169,39,93,38,92,210,168,36,94,208,170,209,171,37,95,219,161,47,85,46,84,218,160,44,86,216,162,217,163,45,87,40,82,220,166,221,167,41,83,223,165,43,81,42,80,222,164,0,123,246,141,241,138,7,124,255,132,9,114,14,117,248,131,227,152,21,110,18,105,228,159,28,103,234,145,237,150,27,96,219,160,45,86,42,81,220,167,36,95,210,169,213,174,35,88,56,67,206,181,201,178,63,68,199,188,49,74,54,77,192,187,171,208,93,38,90,33,172,215,84,47,162,217,165,222,83,40,72,51,190,197,185,194,79,52,183,204,65,58,70,61,176,203,112,11,134,253,129,250,119,12,143,244,121,2,126,5,136,243,147,232,101,30,98,25,148,239,108,23,154,225,157,230,107,16,75,48,189,198,186,193,76,55,180,207,66,57,69,62,179,200,168,211,94,37,89,34,175,212,87,44,161,218,166,221,80,43,144,235,102,29,97,26,151,236,111,20,153,226,158,229,104,19,115,8,133,254,130,249,116,15,140,247,122,1,125,6,139,240,224,155,22,109,17,106,231,156,31,100,233,146,238,149,24,99,3,120,245,142,242,137,4,127,252,135,10,113,13,118,251,128,59,64,205,182,202,177,60,71,196,191,50,73,53,78,195,184,216,163,46,85,41,82,223,164,39,92,209,170,214,173,32,91,0,124,248,132,237,145,21,105,199,187,63,67,42,86,210,174,147,239,107,23,126,2,134,250,84,40,172,208,185,197,65,61,59,71,195,191,214,170,46,82,252,128,4,120,17,109,233,149,168,212,80,44,69,57,189,193,111,19,151,235,130,254,122,6,118,10,142,242,155,231,99,31,177,205,73,53,92,32,164,216,229,153,29,97,8,116,240,140,34,94,218,166,207,179,55,75,77,49,181,201,160,220,88,36,138,246,114,14,103,27,159,227,222,162,38,90,51,79,203,183,25,101,225,157,244,136,12,112,236,144,20,104,1,125,249,133,43,87,211,175,198,186,62,66,127,3,135,251,146,238,106,22,184,196,64,60,85,41,173,209,215,171,47,83,58,70,194,190,16,108,232,148,253,129,5,121,68,56,188,192,169,213,81,45,131,255,123,7,110,18,150,234,154,230,98,30,119,11,143,243,93,33,165,217,176,204,72,52,9,117,241,141,228,152,28,96,206,178,54,74,35,95,219,167,161,221,89,37,76,48,180,200,102,26,158,226,139,247,115,15,50,78,202,182,223,163,39,91,245,137,13,113,24,100,224,156,0,125,250,135,233,148,19,110,207,178,53,72,38,91,220,161,131,254,121,4,106,23,144,237,76,49,182,203,165,216,95,34,27,102,225,156,242,143,8,117,212,169,46,83,61,64,199,186,152,229,98,31,113,12,139,246,87,42,173,208,190,195,68,57,54,75,204,177,223,162,37,88,249,132,3,126,16,109,234,151,181,200,79,50,92,33,166,219,122,7,128,253,147,238,105,20,45,80,215,170,196,185,62,67,226,159,24,101,11,118,241,140,174,211,84,41,71,58,189,192,97,28,155,230,136,245,114,15,108,17,150,235,133,248,127,2,163,222,89,36,74,55,176,205,239,146,21,104,6,123,252,129,32,93,218,167,201,180,51,78,119,10,141,240,158,227,100,25,184,197,66,63,81,44,171,214,244,137,14,115,29,96,231,154,59,70,193,188,210,175,40,85,90,39,160,221,179,206,73,52,149,232,111,18,124,1,134,251,217,164,35,94,48,77,202,183,22,107,236,145,255,130,5,120,65,60,187,198,168,213,82,47,142,243,116,9,103,26,157,224,194,191,56,69,43,86,209,172,13,112,247,138,228,153,30,99,0,126,252,130,229,155,25,103,215,169,43,85,50,76,206,176,179,205,79,49,86,40,170,212,100,26,152,230,129,255,125,3,123,5,135,249,158,224,98,28,172,210,80,46,73,55,181,203,200,182,52,74,45,83,209,175,31,97,227,157,250,132,6,120,246,136,10,116,19,109,239,145,33,95,221,163,196,186,56,70,69,59,185,199,160,222,92,34,146,236,110,16,119,9,139,245,141,243,113,15,104,22,148,234,90,36,166,216,191,193,67,61,62,64,194,188,219,165,39,89,233,151,21,107,12,114,240,142,241,143,13,115,20,106,232,150,38,88,218,164,195,189,63,65,66,60,190,192,167,217,91,37,149,235,105,23,112,14,140,242,138,244,118,8,111,17,147,237,93,35,161,223,184,198,68,58,57,71,197,187,220,162,32,94,238,144,18,108,11,117,247,137,7,121,251,133,226,156,30,96,208,174,44,82,53,75,201,183,180,202,72,54,81,47,173,211,99,29,159,225,134,248,122,4,124,2,128,254,153,231,101,27,171,213,87,41,78,48,178,204,207,177,51,77,42,84,214,168,24,102,228,154,253,131,1,127,0,127,254,129,225,158,31,96,223,160,33,94,62,65,192,191,163,220,93,34,66,61,188,195,124,3,130,253,157,226,99,28,91,36,165,218,186,197,68,59,132,251,122,5,101,26,155,228,248,135,6,121,25,102,231,152,39,88,217,166,198,185,56,71,182,201,72,55,87,40,169,214,105,22,151,232,136,247,118,9,21,106,235,148,244,139,10,117,202,181,52,75,43,84,213,170,237,146,19,108,12,115,242,141,50,77,204,179,211,172,45,82,78,49,176,207,175,208,81,46,145,238,111,16,112,15,142,241,113,14,143,240,144,239,110,17,174,209,80,47,79,48,177,206,210,173,44,83,51,76,205,178,13,114,243,140,236,147,18,109,42,85,212,171,203,180,53,74,245,138,11,116,20,107,234,149,137,246,119,8,104,23,150,233,86,41,168,215,183,200,73,54,199,184,57,70,38,89,216,167,24,103,230,153,249,134,7,120,100,27,154,229,133,250,123,4,187,196,69,58,90,37,164,219,156,227,98,29,125,2,131,252,67,60,189,194,162,221,92,35,63,64,193,190,222,161,32,95,224,159,30,97,1,126,255,128,0,128,29,157,58,186,39,167,116,244,105,233,78,206,83,211,232,104,245,117,210,82,207,79,156,28,129,1,166,38,187,59,205,77,208,80,247,119,234,106,185,57,164,36,131,3,158,30,37,165,56,184,31,159,2,130,81,209,76,204,107,235,118,246,135,7,154,26,189,61,160,32,243,115,238,110,201,73,212,84,111,239,114,242,85,213,72,200,27,155,6,134,33,161,60,188,74,202,87,215,112,240,109,237,62,190,35,163,4,132,25,153,162,34,191,63,152,24,133,5,214,86,203,75,236,108,241,113,19,147,14,142,41,169,52,180,103,231,122,250,93,221,64,192,251,123,230,102,193,65,220,92,143,15,146,18,181,53,168,40,222,94,195,67,228,100,249,121,170,42,183,55,144,16,141,13,54,182,43,171,12,140,17,145,66,194,95,223,120,248,101,229,148,20,137,9,174,46,179,51,224,96,253,125,218,90,199,71,124,252,97,225,70,198,91,219,8,136,21,149,50,178,47,175,89,217,68,196,99,227,126,254,45,173,48,176,23,151,10,138,177,49,172,44,139,11,150,22,197,69,216,88,255,127,226,98,0,129,31,158,62,191,33,160,124,253,99,226,66,195,93,220,248,121,231,102,198,71,217,88,132,5,155,26,186,59,165,36,237,108,242,115,211,82,204,77,145,16,142,15,175,46,176,49,21,148,10,139,43,170,52,181,105,232,118,247,87,214,72,201,199,70,216,89,249,120,230,103,187,58,164,37,133,4,154,27,63,190,32,161,1,128,30,159,67,194,92,221,125,252,98,227,42,171,53,180,20,149,11,138,86,215,73,200,104,233,119,246,210,83,205,76,236,109,243,114,174,47,177,48,144,17,143,14,147,18,140,13,173,44,178,51,239,110,240,113,209,80,206,79,107,234,116,245,85,212,74,203,23,150,8,137,41,168,54,183,126,255,97,224,64,193,95,222,2,131,29,156,60,189,35,162,134,7,153,24,184,57,167,38,250,123,229,100,196,69,219,90,84,213,75,202,106,235,117,244,40,169,55,182,22,151,9,136,172,45,179,50,146,19,141,12,208,81,207,78,238,111,241,112,185,56,166,39,135,6,152,25,197,68,218,91,251,122,228,101,65,192,94,223,127,254,96,225,61,188,34,163,3,130,28,157,0,130,25,155,50,176,43,169,100,230,125,255,86,212,79,205,200,74,209,83,250,120,227,97,172,46,181,55,158,28,135,5,141,15,148,22,191,61,166,36,233,107,240,114,219,89,194,64,69,199,92,222,119,245,110,236,33,163,56,186,19,145,10,136,7,133,30,156,53,183,44,174,99,225,122,248,81,211,72,202,207,77,214,84,253,127,228,102,171,41,178,48,153,27,128,2,138,8,147,17,184,58,161,35,238,108,247,117,220,94,197,71,66,192,91,217,112,242,105,235,38,164,63,189,20,150,13,143,14,140,23,149,60,190,37,167,106,232,115,241,88,218,65,195,198,68,223,93,244,118,237,111,162,32,187,57,144,18,137,11,131,1,154,24,177,51,168,42,231,101,254,124,213,87,204,78,75,201,82,208,121,251,96,226,47,173,54,180,29,159,4,134,9,139,16,146,59,185,34,160,109,239,116,246,95,221,70,196,193,67,216,90,243,113,234,104,165,39,188,62,151,21,142,12,132,6,157,31,182,52,175,45,224,98,249,123,210,80,203,73,76,206,85,215,126,252,103,229,40,170,49,179,26,152,3,129,0,131,27,152,54,181,45,174,108,239,119,244,90,217,65,194,216,91,195,64,238,109,245,118,180,55,175,44,130,1,153,26,173,46,182,53,155,24,128,3,193,66,218,89,247,116,236,111,117,246,110,237,67,192,88,219,25,154,2,129,47,172,52,183,71,196,92,223,113,242,106,233,43,168,48,179,29,158,6,133,159,28,132,7,169,42,178,49,243,112,232,107,197,70,222,93,234,105,241,114,220,95,199,68,134,5,157,30,176,51,171,40,50,177,41,170,4,135,31,156,94,221,69,198,104,235,115,240,142,13,149,22,184,59,163,32,226,97,249,122,212,87,207,76,86,213,77,206,96,227,123,248,58,185,33,162,12,143,23,148,35,160,56,187,21,150,14,141,79,204,84,215,121,250,98,225,251,120,224,99,205,78,214,85,151,20,140,15,161,34,186,57,201,74,210,81,255,124,228,103,165,38,190,61,147,16,136,11,17,146,10,137,39,164,60,191,125,254,102,229,75,200,80,211,100,231,127,252,82,209,73,202,8,139,19,144,62,189,37,166,188,63,167,36,138,9,145,18,208,83,203,72,230,101,253,126,0,132,21,145,42,174,63,187,84,208,65,197,126,250,107,239,168,44,189,57,130,6,151,19,252,120,233,109,214,82,195,71,77,201,88,220,103,227,114,246,25,157,12,136,51,183,38,162,229,97,240,116,207,75,218,94,177,53,164,32,155,31,142,10,154,30,143,11,176,52,165,33,206,74,219,95,228,96,241,117,50,182,39,163,24,156,13,137,102,226,115,247,76,200,89,221,215,83,194,70,253,121,232,108,131,7,150,18,169,45,188,56,127,251,106,238,85,209,64,196,43,175,62,186,1,133,20,144,41,173,60,184,3,135,22,146,125,249,104,236,87,211,66,198,129,5,148,16,171,47,190,58,213,81,192,68,255,123,234,110,100,224,113,245,78,202,91,223,48,180,37,161,26,158,15,139,204,72,217,93,230,98,243,119,152,28,141,9,178,54,167,35,179,55,166,34,153,29,140,8,231,99,242,118,205,73,216,92,27,159,14,138,49,181,36,160,79,203,90,222,101,225,112,244,254,122,235,111,212,80,193,69,170,46,191,59,128,4,149,17,86,210,67,199,124,248,105,237,2,134,23,147,40,172,61,185,0,133,23,146,46,171,57,188,92,217,75,206,114,247,101,224,184,61,175,42,150,19,129,4,228,97,243,118,202,79,221,88,109,232,122,255,67,198,84,209,49,180,38,163,31,154,8,141,213,80,194,71,251,126,236,105,137,12,158,27,167,34,176,53,218,95,205,72,244,113,227,102,134,3,145,20,168,45,191,58,98,231,117,240,76,201,91,222,62,187,41,172,16,149,7,130,183,50,160,37,153,28,142,11,235,110,252,121,197,64,210,87,15,138,24,157,33,164,54,179,83,214,68,193,125,248,106,239,169,44,190,59,135,2,144,21,245,112,226,103,219,94,204,73,17,148,6,131,63,186,40,173,77,200,90,223,99,230,116,241,196,65,211,86,234,111,253,120,152,29,143,10,182,51,161,36,124,249,107,238,82,215,69,192,32,165,55,178,14,139,25,156,115,246,100,225,93,216,74,207,47,170,56,189,1,132,22,147,203,78,220,89,229,96,242,119,151,18,128,5,185,60,174,43,30,155,9,140,48,181,39,162,66,199,85,208,108,233,123,254,166,35,177,52,136,13,159,26,250,127,237,104,212,81,195,70,0,134,17,151,34,164,51,181,68,194,85,211,102,224,119,241,136,14,153,31,170,44,187,61,204,74,221,91,238,104,255,121,13,139,28,154,47,169,62,184,73,207,88,222,107,237,122,252,133,3,148,18,167,33,182,48,193,71,208,86,227,101,242,116,26,156,11,141,56,190,41,175,94,216,79,201,124,250,109,235,146,20,131,5,176,54,161,39,214,80,199,65,244,114,229,99,23,145,6,128,53,179,36,162,83,213,66,196,113,247,96,230,159,25,142,8,189,59,172,42,219,93,202,76,249,127,232,110,52,178,37,163,22,144,7,129,112,246,97,231,82,212,67,197,188,58,173,43,158,24,143,9,248,126,233,111,218,92,203,77,57,191,40,174,27,157,10,140,125,251,108,234,95,217,78,200,177,55,160,38,147,21,130,4,245,115,228,98,215,81,198,64,46,168,63,185,12,138,29,155,106,236,123,253,72,206,89,223,166,32,183,49,132,2,149,19,226,100,243,117,192,70,209,87,35,165,50,180,1,135,16,150,103,225,118,240,69,195,84,210,171,45,186,60,137,15,152,30,239,105,254,120,205,75,220,90,0,135,19,148,38,161,53,178,76,203,95,216,106,237,121,254,152,31,139,12,190,57,173,42,212,83,199,64,242,117,225,102,45,170,62,185,11,140,24,159,97,230,114,245,71,192,84,211,181,50,166,33,147,20,128,7,249,126,234,109,223,88,204,75,90,221,73,206,124,251,111,232,22,145,5,130,48,183,35,164,194,69,209,86,228,99,247,112,142,9,157,26,168,47,187,60,119,240,100,227,81,214,66,197,59,188,40,175,29,154,14,137,239,104,252,123,201,78,218,93,163,36,176,55,133,2,150,17,180,51,167,32,146,21,129,6,248,127,235,108,222,89,205,74,44,171,63,184,10,141,25,158,96,231,115,244,70,193,85,210,153,30,138,13,191,56,172,43,213,82,198,65,243,116,224,103,1,134,18,149,39,160,52,179,77,202,94,217,107,236,120,255,238,105,253,122,200,79,219,92,162,37,177,54,132,3,151,16,118,241,101,226,80,215,67,196,58,189,41,174,28,155,15,136,195,68,208,87,229,98,246,113,143,8,156,27,169,46,186,61,91,220,72,207,125,250,110,233,23,144,4,131,49,182,34,165,0,136,13,133,26,146,23,159,52,188,57,177,46,166,35,171,104,224,101,237,114,250,127,247,92,212,81,217,70,206,75,195,208,88,221,85,202,66,199,79,228,108,233,97,254,118,243,123,184,48,181,61,162,42,175,39,140,4,129,9,150,30,155,19,189,53,176,56,167,47,170,34,137,1,132,12,147,27,158,22,213,93,216,80,207,71,194,74,225,105,236,100,251,115,246,126,109,229,96,232,119,255,122,242,89,209,84,220,67,203,78,198,5,141,8,128,31,151,18,154,49,185,60,180,43,163,38,174,103,239,106,226,125,245,112,248,83,219,94,214,73,193,68,204,15,135,2,138,21,157,24,144,59,179,54,190,33,169,44,164,183,63,186,50,173,37,160,40,131,11,142,6,153,17,148,28,223,87,210,90,197,77,200,64,235,99,230,110,241,121,252,116,218,82,215,95,192,72,205,69,238,102,227,107,244,124,249,113,178,58,191,55,168,32,165,45,134,14,139,3,156,20,145,25,10,130,7,143,16,152,29,149,62,182,51,187,36,172,41,161,98,234,111,231,120,240,117,253,86,222,91,211,76,196,65,201,0,137,15,134,30,151,17,152,60,181,51,186,34,171,45,164,120,241,119,254,102,239,105,224,68,205,75,194,90,211,85,220,240,121,255,118,238,103,225,104,204,69,195,74,210,91,221,84,136,1,135,14,150,31,153,16,180,61,187,50,170,35,165,44,253,116,242,123,227,106,236,101,193,72,206,71,223,86,208,89,133,12,138,3,155,18,148,29,185,48,182,63,167,46,168,33,13,132,2,139,19,154,28,149,49,184,62,183,47,166,32,169,117,252,122,243,107,226,100,237,73,192,70,207,87,222,88,209,231,110,232,97,249,112,246,127,219,82,212,93,197,76,202,67,159,22,144,25,129,8,142,7,163,42,172,37,189,52,178,59,23,158,24,145,9,128,6,143,43,162,36,173,53,188,58,179,111,230,96,233,113,248,126,247,83,218,92,213,77,196,66,203,26,147,21,156,4,141,11,130,38,175,41,160,56,177,55,190,98,235,109,228,124,245,115,250,94,215,81,216,64,201,79,198,234,99,229,108,244,125,251,114,214,95,217,80,200,65,199,78,146,27,157,20,140,5,131,10,174,39,161,40,176,57,191,54,0,138,9,131,18,152,27,145,36,174,45,167,54,188,63,181,72,194,65,203,90,208,83,217,108,230,101,239,126,244,119,253,144,26,153,19,130,8,139,1,180,62,189,55,166,44,175,37,216,82,209,91,202,64,195,73,252,118,245,127,238,100,231,109,61,183,52,190,47,165,38,172,25,147,16,154,11,129,2,136,117,255,124,246,103,237,110,228,81,219,88,210,67,201,74,192,173,39,164,46,191,53,182,60,137,3,128,10,155,17,146,24,229,111,236,102,247,125,254,116,193,75,200,66,211,89,218,80,122,240,115,249,104,226,97,235,94,212,87,221,76,198,69,207,50,184,59,177,32,170,41,163,22,156,31,149,4,142,13,135,234,96,227,105,248,114,241,123,206,68,199,77,220,86,213,95,162,40,171,33,176,58,185,51,134,12,143,5,148,30,157,23,71,205,78,196,85,223,92,214,99,233,106,224,113,251,120,242,15,133,6,140,29,151,20,158,43,161,34,168,57,179,48,186,215,93,222,84,197,79,204,70,243,121,250,112,225,107,232,98,159,21,150,28,141,7,132,14,187,49,178,56,169,35,160,42,0,139,11,128,22,157,29,150,44,167,39,172,58,177,49,186,88,211,83,216,78,197,69,206,116,255,127,244,98,233,105,226,176,59,187,48,166,45,173,38,156,23,151,28,138,1,129,10,232,99,227,104,254,117,245,126,196,79,207,68,210,89,217,82,125,246,118,253,107,224,96,235,81,218,90,209,71,204,76,199,37,174,46,165,51,184,56,179,9,130,2,137,31,148,20,159,205,70,198,77,219,80,208,91,225,106,234,97,247,124,252,119,149,30,158,21,131,8,136,3,185,50,178,57,175,36,164,47,250,113,241,122,236,103,231,108,214,93,221,86,192,75,203,64,162,41,169,34,180,63,191,52,142,5,133,14,152,19,147,24,74,193,65,202,92,215,87,220,102,237,109,230,112,251,123,240,18,153,25,146,4,143,15,132,62,181,53,190,40,163,35,168,135,12,140,7,145,26,154,17,171,32,160,43,189,54,182,61,223,84,212,95,201,66,194,73,243,120,248,115,229,110,238,101,55,188,60,183,33,170,42,161,27,144,16,155,13,134,6,141,111,228,100,239,121,242,114,249,67,200,72,195,85,222,94,213,0,140,5,137,10,134,15,131,20,152,17,157,30,146,27,151,40,164,45,161,34,174,39,171,60,176,57,181,54,186,51,191,80,220,85,217,90,214,95,211,68,200,65,205,78,194,75,199,120,244,125,241,114,254,119,251,108,224,105,229,102,234,99,239,160,44,165,41,170,38,175,35,180,56,177,61,190,50,187,55,136,4,141,1,130,14,135,11,156,16,153,21,150,26,147,31,240,124,245,121,250,118,255,115,228,104,225,109,238,98,235,103,216,84,221,81,210,94,215,91,204,64,201,69,198,74,195,79,93,209,88,212,87,219,82,222,73,197,76,192,67,207,70,202,117,249,112,252,127,243,122,246,97,237,100,232,107,231,110,226,13,129,8,132,7,139,2,142,25,149,28,144,19,159,22,154,37,169,32,172,47,163,42,166,49,189,52,184,59,183,62,178,253,113,248,116,247,123,242,126,233,101,236,96,227,111,230,106,213,89,208,92,223,83,218,86,193,77,196,72,203,71,206,66,173,33,168,36,167,43,162,46,185,53,188,48,179,63,182,58,133,9,128,12,143,3,138,6,145,29,148,24,155,23,158,18,0,141,7,138,14,131,9,132,28,145,27,150,18,159,21,152,56,181,63,178,54,187,49,188,36,169,35,174,42,167,45,160,112,253,119,250,126,243,121,244,108,225,107,230,98,239,101,232,72,197,79,194,70,203,65,204,84,217,83,222,90,215,93,208,224,109,231,106,238,99,233,100,252,113,251,118,242,127,245,120,216,85,223,82,214,91,209,92,196,73,195,78,202,71,205,64,144,29,151,26,158,19,153,20,140,1,139,6,130,15,133,8,168,37,175,34,166,43,161,44,180,57,179,62,186,55,189,48,221,80,218,87,211,94,212,89,193,76,198,75,207,66,200,69,229,104,226,111,235,102,236,97,249,116,254,115,247,122,240,125,173,32,170,39,163,46,164,41,177,60,182,59,191,50,184,53,149,24,146,31,155,22,156,17,137,4,142,3,135,10,128,13,61,176,58,183,51,190,52,185,33,172,38,171,47,162,40,165,5,136,2,143,11,134,12,129,25,148,30,147,23,154,16,157,77,192,74,199,67,206,68,201,81,220,86,219,95,210,88,213,117,248,114,255,123,246,124,241,105,228,110,227,103,234,96,237,0,142,1,143,2,140,3,141,4,138,5,139,6,136,7,137,8,134,9,135,10,132,11,133,12,130,13,131,14,128,15,129,16,158,17,159,18,156,19,157,20,154,21,155,22,152,23,153,24,150,25,151,26,148,27,149,28,146,29,147,30,144,31,145,32,174,33,175,34,172,35,173,36,170,37,171,38,168,39,169,40,166,41,167,42,164,43,165,44,162,45,163,46,160,47,161,48,190,49,191,50,188,51,189,52,186,53,187,54,184,55,185,56,182,57,183,58,180,59,181,60,178,61,179,62,176,63,177,64,206,65,207,66,204,67,205,68,202,69,203,70,200,71,201,72,198,73,199,74,196,75,197,76,194,77,195,78,192,79,193,80,222,81,223,82,220,83,221,84,218,85,219,86,216,87,217,88,214,89,215,90,212,91,213,92,210,93,211,94,208,95,209,96,238,97,239,98,236,99,237,100,234,101,235,102,232,103,233,104,230,105,231,106,228,107,229,108,226,109,227,110,224,111,225,112,254,113,255,114,252,115,253,116,250,117,251,118,248,119,249,120,246,121,247,122,244,123,245,124,242,125,243,126,240,127,241,0,143,3,140,6,137,5,138,12,131,15,128,10,133,9,134,24,151,27,148,30,145,29,146,20,155,23,152,18,157,17,158,48,191,51,188,54,185,53,186,60,179,63,176,58,181,57,182,40,167,43,164,46,161,45,162,36,171,39,168,34,173,33,174,96,239,99,236,102,233,101,234,108,227,111,224,106,229,105,230,120,247,123,244,126,241,125,242,116,251,119,248,114,253,113,254,80,223,83,220,86,217,85,218,92,211,95,208,90,213,89,214,72,199,75,196,78,193,77,194,68,203,71,200,66,205,65,206,192,79,195,76,198,73,197,74,204,67,207,64,202,69,201,70,216,87,219,84,222,81,221,82,212,91,215,88,210,93,209,94,240,127,243,124,246,121,245,122,252,115,255,112,250,117,249,118,232,103,235,100,238,97,237,98,228,107,231,104,226,109,225,110,160,47,163,44,166,41,165,42,172,35,175,32,170,37,169,38,184,55,187,52,190,49,189,50,180,59,183,56,178,61,177,62,144,31,147,28,150,25,149,26,156,19,159,16,154,21,153,22,136,7,139,4,142,1,141,2,132,11,135,8,130,13,129,14,0,144,61,173,122,234,71,215,244,100,201,89,142,30,179,35,245,101,200,88,143,31,178,34,1,145,60,172,123,235,70,214,247,103,202,90,141,29,176,32,3,147,62,174,121,233,68,212,2,146,63,175,120,232,69,213,246,102,203,91,140,28,177,33,243,99,206,94,137,25,180,36,7,151,58,170,125,237,64,208,6,150,59,171,124,236,65,209,242,98,207,95,136,24,181,37,4,148,57,169,126,238,67,211,240,96,205,93,138,26,183,39,241,97,204,92,139,27,182,38,5,149,56,168,127,239,66,210,251,107,198,86,129,17,188,44,15,159,50,162,117,229,72,216,14,158,51,163,116,228,73,217,250,106,199,87,128,16,189,45,12,156,49,161,118,230,75,219,248,104,197,85,130,18,191,47,249,105,196,84,131,19,190,46,13,157,48,160,119,231,74,218,8,152,53,165,114,226,79,223,252,108,193,81,134,22,187,43,253,109,192,80,135,23,186,42,9,153,52,164,115,227,78,222,255,111,194,82,133,21,184,40,11,155,54,166,113,225,76,220,10,154,55,167,112,224,77,221,254,110,195,83,132,20,185,41,0,145,63,174,126,239,65,208,252,109,195,82,130,19,189,44,229,116,218,75,155,10,164,53,25,136,38,183,103,246,88,201,215,70,232,121,169,56,150,7,43,186,20,133,85,196,106,251,50,163,13,156,76,221,115,226,206,95,241,96,176,33,143,30,179,34,140,29,205,92,242,99,79,222,112,225,49,160,14,159,86,199,105,248,40,185,23,134,170,59,149,4,212,69,235,122,100,245,91,202,26,139,37,180,152,9,167,54,230,119,217,72,129,16,190,47,255,110,192,81,125,236,66,211,3,146,60,173,123,234,68,213,5,148,58,171,135,22,184,41,249,104,198,87,158,15,161,48,224,113,223,78,98,243,93,204,28,141,35,178,172,61,147,2,210,67,237,124,80,193,111,254,46,191,17,128,73,216,118,231,55,166,8,153,181,36,138,27,203,90,244,101,200,89,247,102,182,39,137,24,52,165,11,154,74,219,117,228,45,188,18,131,83,194,108,253,209,64,238,127,175,62,144,1,31,142,32,177,97,240,94,207,227,114,220,77,157,12,162,51,250,107,197,84,132,21,187,42,6,151,57,168,120,233,71,214,0,146,57,171,114,224,75,217,228,118,221,79,150,4,175,61,213,71,236,126,167,53,158,12,49,163,8,154,67,209,122,232,183,37,142,28,197,87,252,110,83,193,106,248,33,179,24,138,98,240,91,201,16,130,41,187,134,20,191,45,244,102,205,95,115,225,74,216,1,147,56,170,151,5,174,60,229,119,220,78,166,52,159,13,212,70,237,127,66,208,123,233,48,162,9,155,196,86,253,111,182,36,143,29,32,178,25,139,82,192,107,249,17,131,40,186,99,241,90,200,245,103,204,94,135,21,190,44,230,116,223,77,148,6,173,63,2,144,59,169,112,226,73,219,51,161,10,152,65,211,120,234,215,69,238,124,165,55,156,14,81,195,104,250,35,177,26,136,181,39,140,30,199,85,254,108,132,22,189,47,246,100,207,93,96,242,89,203,18,128,43,185,149,7,172,62,231,117,222,76,113,227,72,218,3,145,58,168,64,210,121,235,50,160,11,153,164,54,157,15,214,68,239,125,34,176,27,137,80,194,105,251,198,84,255,109,180,38,141,31,247,101,206,92,133,23,188,46,19,129,42,184,97,243,88,202,0,147,59,168,118,229,77,222,236,127,215,68,154,9,161,50,197,86,254,109,179,32,136,27,41,186,18,129,95,204,100,247,151,4,172,63,225,114,218,73,123,232,64,211,13,158,54,165,82,193,105,250,36,183,31,140,190,45,133,22,200,91,243,96,51,160,8,155,69,214,126,237,223,76,228,119,169,58,146,1,246,101,205,94,128,19,187,40,26,137,33,178,108,255,87,196,164,55,159,12,210,65,233,122,72,219,115,224,62,173,5,150,97,242,90,201,23,132,44,191,141,30,182,37,251,104,192,83,102,245,93,206,16,131,43,184,138,25,177,34,252,111,199,84,163,48,152,11,213,70,238,125,79,220,116,231,57,170,2,145,241,98,202,89,135,20,188,47,29,142,38,181,107,248,80,195,52,167,15,156,66,209,121,234,216,75,227,112,174,61,149,6,85,198,110,253,35,176,24,139,185,42,130,17,207,92,244,103,144,3,171,56,230,117,221,78,124,239,71,212,10,153,49,162,194,81,249,106,180,39,143,28,46,189,21,134,88,203,99,240,7,148,60,175,113,226,74,217,235,120,208,67,157,14,166,53,0,148,53,161,106,254,95,203,212,64,225,117,190,42,139,31,181,33,128,20,223,75,234,126,97,245,84,192,11,159,62,170,119,227,66,214,29,137,40,188,163,55,150,2,201,93,252,104,194,86,247,99,168,60,157,9,22,130,35,183,124,232,73,221,238,122,219,79,132,16,177,37,58,174,15,155,80,196,101,241,91,207,110,250,49,165,4,144,143,27,186,46,229,113,208,68,153,13,172,56,243,103,198,82,77,217,120,236,39,179,18,134,44,184,25,141,70,210,115,231,248,108,205,89,146,6,167,51,193,85,244,96,171,63,158,10,21,129,32,180,127,235,74,222,116,224,65,213,30,138,43,191,160,52,149,1,202,94,255,107,182,34,131,23,220,72,233,125,98,246,87,195,8,156,61,169,3,151,54,162,105,253,92,200,215,67,226,118,189,41,136,28,47,187,26,142,69,209,112,228,251,111,206,90,145,5,164,48,154,14,175,59,240,100,197,81,78,218,123,239,36,176,17,133,88,204,109,249,50,166,7,147,140,24,185,45,230,114,211,71,237,121,216,76,135,19,178,38,57,173,12,152,83,199,102,242,0,149,55,162,110,251,89,204,220,73,235,126,178,39,133,16,165,48,146,7,203,94,252,105,121,236,78,219,23,130,32,181,87,194,96,245,57,172,14,155,139,30,188,41,229,112,210,71,242,103,197,80,156,9,171,62,46,187,25,140,64,213,119,226,174,59,153,12,192,85,247,98,114,231,69,208,28,137,43,190,11,158,60,169,101,240,82,199,215,66,224,117,185,44,142,27,249,108,206,91,151,2,160,53,37,176,18,135,75,222,124,233,92,201,107,254,50,167,5,144,128,21,183,34,238,123,217,76,65,212,118,227,47,186,24,141,157,8,170,63,243,102,196,81,228,113,211,70,138,31,189,40,56,173,15,154,86,195,97,244,22,131,33,180,120,237,79,218,202,95,253,104,164,49,147,6,179,38,132,17,221,72,234,127,111,250,88,205,1,148,54,163,239,122,216,77,129,20,182,35,51,166,4,145,93,200,106,255,74,223,125,232,36,177,19,134,150,3,161,52,248,109,207,90,184,45,143,26,214,67,225,116,100,241,83,198,10,159,61,168,29,136,42,191,115,230,68,209,193,84,246,99,175,58,152,13,0,150,49,167,98,244,83,197,196,82,245,99,166,48,151,1,149,3,164,50,247,97,198,80,81,199,96,246,51,165,2,148,55,161,6,144,85,195,100,242,243,101,194,84,145,7,160,54,162,52,147,5,192,86,241,103,102,240,87,193,4,146,53,163,110,248,95,201,12,154,61,171,170,60,155,13,200,94,249,111,251,109,202,92,153,15,168,62,63,169,14,152,93,203,108,250,89,207,104,254,59,173,10,156,157,11,172,58,255,105,206,88,204,90,253,107,174,56,159,9,8,158,57,175,106,252,91,205,220,74,237,123,190,40,143,25,24,142,41,191,122,236,75,221,73,223,120,238,43,189,26,140,141,27,188,42,239,121,222,72,235,125,218,76,137,31,184,46,47,185,30,136,77,219,124,234,126,232,79,217,28,138,45,187,186,44,139,29,216,78,233,127,178,36,131,21,208,70,225,119,118,224,71,209,20,130,37,179,39,177,22,128,69,211,116,226,227,117,210,68,129,23,176,38,133,19,180,34,231,113,214,64,65,215,112,230,35,181,18,132,16,134,33,183,114,228,67,213,212,66,229,115,182,32,135,17,0,151,51,164,102,241,85,194,204,91,255,104,170,61,153,14,133,18,182,33,227,116,208,71,73,222,122,237,47,184,28,139,23,128,36,179,113,230,66,213,219,76,232,127,189,42,142,25,146,5,161,54,244,99,199,80,94,201,109,250,56,175,11,156,46,185,29,138,72,223,123,236,226,117,209,70,132,19,183,32,171,60,152,15,205,90,254,105,103,240,84,195,1,150,50,165,57,174,10,157,95,200,108,251,245,98,198,81,147,4,160,55,188,43,143,24,218,77,233,126,112,231,67,212,22,129,37,178,92,203,111,248,58,173,9,158,144,7,163,52,246,97,197,82,217,78,234,125,191,40,140,27,21,130,38,177,115,228,64,215,75,220,120,239,45,186,30,137,135,16,180,35,225,118,210,69,206,89,253,106,168,63,155,12,2,149,49,166,100,243,87,192,114,229,65,214,20,131,39,176,190,41,141,26,216,79,235,124,247,96,196,83,145,6,162,53,59,172,8,159,93,202,110,249,101,242,86,193,3,148,48,167,169,62,154,13,207,88,252,107,224,119,211,68,134,17,181,34,44,187,31,136,74,221,121,238,0,152,45,181,90,194,119,239,180,44,153,1,238,118,195,91,117,237,88,192,47,183,2,154,193,89,236,116,155,3,182,46,234,114,199,95,176,40,157,5,94,198,115,235,4,156,41,177,159,7,178,42,197,93,232,112,43,179,6,158,113,233,92,196,201,81,228,124,147,11,190,38,125,229,80,200,39,191,10,146,188,36,145,9,230,126,203,83,8,144,37,189,82,202,127,231,35,187,14,150,121,225,84,204,151,15,186,34,205,85,224,120,86,206,123,227,12,148,33,185,226,122,207,87,184,32,149,13,143,23,162,58,213,77,248,96,59,163,22,142,97,249,76,212,250,98,215,79,160,56,141,21,78,214,99,251,20,140,57,161,101,253,72,208,63,167,18,138,209,73,252,100,139,19,166,62,16,136,61,165,74,210,103,255,164,60,137,17,254,102,211,75,70,222,107,243,28,132,49,169,242,106,223,71,168,48,133,29,51,171,30,134,105,241,68,220,135,31,170,50,221,69,240,104,172,52,129,25,246,110,219,67,24,128,53,173,66,218,111,247,217,65,244,108,131,27,174,54,109,245,64,216,55,175,26,130,0,153,47,182,94,199,113,232,188,37,147,10,226,123,205,84,101,252,74,211,59,162,20,141,217,64,246,111,135,30,168,49,202,83,229,124,148,13,187,34,118,239,89,192,40,177,7,158,175,54,128,25,241,104,222,71,19,138,60,165,77,212,98,251,137,16,166,63,215,78,248,97,53,172,26,131,107,242,68,221,236,117,195,90,178,43,157,4,80,201,127,230,14,151,33,184,67,218,108,245,29,132,50,171,255,102,208,73,161,56,142,23,38,191,9,144,120,225,87,206,154,3,181,44,196,93,235,114,15,150,32,185,81,200,126,231,179,42,156,5,237,116,194,91,106,243,69,220,52,173,27,130,214,79,249,96,136,17,167,62,197,92,234,115,155,2,180,45,121,224,86,207,39,190,8,145,160,57,143,22,254,103,209,72,28,133,51,170,66,219,109,244,134,31,169,48,216,65,247,110,58,163,21,140,100,253,75,210,227,122,204,85,189,36,146,11,95,198,112,233,1,152,46,183,76,213,99,250,18,139,61,164,240,105,223,70,174,55,129,24,41,176,6,159,119,238,88,193,149,12,186,35,203,82,228,125,0,154,41,179,82,200,123,225,164,62,141,23,246,108,223,69,85,207,124,230,7,157,46,180,241,107,216,66,163,57,138,16,170,48,131,25,248,98,209,75,14,148,39,189,92,198,117,239,255,101,214,76,173,55,132,30,91,193,114,232,9,147,32,186,73,211,96,250,27,129,50,168,237,119,196,94,191,37,150,12,28,134,53,175,78,212,103,253,184,34,145,11,234,112,195,89,227,121,202,80,177,43,152,2,71,221,110,244,21,143,60,166,182,44,159,5,228,126,205,87,18,136,59,161,64,218,105,243,146,8,187,33,192,90,233,115,54,172,31,133,100,254,77,215,199,93,238,116,149,15,188,38,99,249,74,208,49,171,24,130,56,162,17,139,106,240,67,217,156,6,181,47,206,84,231,125,109,247,68,222,63,165,22,140,201,83,224,122,155,1,178,40,219,65,242,104,137,19,160,58,127,229,86,204,45,183,4,158,142,20,167,61,220,70,245,111,42,176,3,153,120,226,81,203,113,235,88,194,35,185,10,144,213,79,252,102,135,29,174,52,36,190,13,151,118,236,95,197,128,26,169,51,210,72,251,97,0,155,43,176,86,205,125,230,172,55,135,28,250,97,209,74,69,222,110,245,19,136,56,163,233,114,194,89,191,36,148,15,138,17,161,58,220,71,247,108,38,189,13,150,112,235,91,192,207,84,228,127,153,2,178,41,99,248,72,211,53,174,30,133,9,146,34,185,95,196,116,239,165,62,142,21,243,104,216,67,76,215,103,252,26,129,49,170,224,123,203,80,182,45,157,6,131,24,168,51,213,78,254,101,47,180,4,159,121,226,82,201,198,93,237,118,144,11,187,32,106,241,65,218,60,167,23,140,18,137,57,162,68,223,111,244,190,37,149,14,232,115,195,88,87,204,124,231,1,154,42,177,251,96,208,75,173,54,134,29,152,3,179,40,206,85,229,126,52,175,31,132,98,249,73,210,221,70,246,109,139,16,160,59,113,234,90,193,39,188,12,151,27,128,48,171,77,214,102,253,183,44,156,7,225,122,202,81,94,197,117,238,8,147,35,184,242,105,217,66,164,63,143,20,145,10,186,33,199,92,236,119,61,166,22,141,107,240,64,219,212,79,255,100,130,25,169,50,120,227,83,200,46,181,5,158,0,156,37,185,74,214,111,243,148,8,177,45,222,66,251,103,53,169,16,140,127,227,90,198,161,61,132,24,235,119,206,82,106,246,79,211,32,188,5,153,254,98,219,71,180,40,145,13,95,195,122,230,21,137,48,172,203,87,238,114,129,29,164,56,212,72,241,109,158,2,187,39,64,220,101,249,10,150,47,179,225,125,196,88,171,55,142,18,117,233,80,204,63,163,26,134,190,34,155,7,244,104,209,77,42,182,15,147,96,252,69,217,139,23,174,50,193,93,228,120,31,131,58,166,85,201,112,236,181,41,144,12,255,99,218,70,33,189,4,152,107,247,78,210,128,28,165,57,202,86,239,115,20,136,49,173,94,194,123,231,223,67,250,102,149,9,176,44,75,215,110,242,1,157,36,184,234,118,207,83,160,60,133,25,126,226,91,199,52,168,17,141,97,253,68,216,43,183,14,146,245,105,208,76,191,35,154,6,84,200,113,237,30,130,59,167,192,92,229,121,138,22,175,51,11,151,46,178,65,221,100,248,159,3,186,38,213,73,240,108,62,162,27,135,116,232,81,205,170,54,143,19,224,124,197,89,0,157,39,186,78,211,105,244,156,1,187,38,210,79,245,104,37,184,2,159,107,246,76,209,185,36,158,3,247,106,208,77,74,215,109,240,4,153,35,190,214,75,241,108,152,5,191,34,111,242,72,213,33,188,6,155,243,110,212,73,189,32,154,7,148,9,179,46,218,71,253,96,8,149,47,178,70,219,97,252,177,44,150,11,255,98,216,69,45,176,10,151,99,254,68,217,222,67,249,100,144,13,183,42,66,223,101,248,12,145,43,182,251,102,220,65,181,40,146,15,103,250,64,221,41,180,14,147,53,168,18,143,123,230,92,193,169,52,142,19,231,122,192,93,16,141,55,170,94,195,121,228,140,17,171,54,194,95,229,120,127,226,88,197,49,172,22,139,227,126,196,89,173,48,138,23,90,199,125,224,20,137,51,174,198,91,225,124,136,21,175,50,161,60,134,27,239,114,200,85,61,160,26,135,115,238,84,201,132,25,163,62,202,87,237,112,24,133,63,162,86,203,113,236,235,118,204,81,165,56,130,31,119,234,80,205,57,164,30,131,206,83,233,116,128,29,167,58,82,207,117,232,28,129,59,166,0,158,33,191,66,220,99,253,132,26,165,59,198,88,231,121,21,139,52,170,87,201,118,232,145,15,176,46,211,77,242,108,42,180,11,149,104,246,73,215,174,48,143,17,236,114,205,83,63,161,30,128,125,227,92,194,187,37,154,4,249,103,216,70,84,202,117,235,22,136,55,169,208,78,241,111,146,12,179,45,65,223,96,254,3,157,34,188,197,91,228,122,135,25,166,56,126,224,95,193,60,162,29,131,250,100,219,69,184,38,153,7,107,245,74,212,41,183,8,150,239,113,206,80,173,51,140,18,168,54,137,23,234,116,203,85,44,178,13,147,110,240,79,209,189,35,156,2,255,97,222,64,57,167,24,134,123,229,90,196,130,28,163,61,192,94,225,127,6,152,39,185,68,218,101,251,151,9,182,40,213,75,244,106,19,141,50,172,81,207,112,238,252,98,221,67,190,32,159,1,120,230,89,199,58,164,27,133,233,119,200,86,171,53,138,20,109,243,76,210,47,177,14,144,214,72,247,105,148,10,181,43,82,204,115,237,16,142,49,175,195,93,226,124,129,31,160,62,71,217,102,248,5,155,36,186,0,159,35,188,70,217,101,250,140,19,175,48,202,85,233,118,5,154,38,185,67,220,96,255,137,22,170,53,207,80,236,115,10,149,41,182,76,211,111,240,134,25,165,58,192,95,227,124,15,144,44,179,73,214,106,245,131,28,160,63,197,90,230,121,20,139,55,168,82,205,113,238,152,7,187,36,222,65,253,98,17,142,50,173,87,200,116,235,157,2,190,33,219,68,248,103,30,129,61,162,88,199,123,228,146,13,177,46,212,75,247,104,27,132,56,167,93,194,126,225,151,8,180,43,209,78,242,109,40,183,11,148,110,241,77,210,164,59,135,24,226,125,193,94,45,178,14,145,107,244,72,215,161,62,130,29,231,120,196,91,34,189,1,158,100,251,71,216,174,49,141,18,232,119,203,84,39,184,4,155,97,254,66,221,171,52,136,23,237,114,206,81,60,163,31,128,122,229,89,198,176,47,147,12,246,105,213,74,57,166,26,133,127,224,92,195,181,42,150,9,243,108,208,79,54,169,21,138,112,239,83,204,186,37,153,6,252,99,223,64,51,172,16,143,117,234,86,201,191,32,156,3,249,102,218,69,0,160,93,253,186,26,231,71,105,201,52,148,211,115,142,46,210,114,143,47,104,200,53,149,187,27,230,70,1,161,92,252,185,25,228,68,3,163,94,254,208,112,141,45,106,202,55,151,107,203,54,150,209,113,140,44,2,162,95,255,184,24,229,69,111,207,50,146,213,117,136,40,6,166,91,251,188,28,225,65,189,29,224,64,7,167,90,250,212,116,137,41,110,206,51,147,214,118,139,43,108,204,49,145,191,31,226,66,5,165,88,248,4,164,89,249,190,30,227,67,109,205,48,144,215,119,138,42,222,126,131,35,100,196,57,153,183,23,234,74,13,173,80,240,12,172,81,241,182,22,235,75,101,197,56,152,223,127,130,34,103,199,58,154,221,125,128,32,14,174,83,243,180,20,233,73,181,21,232,72,15,175,82,242,220,124,129,33,102,198,59,155,177,17,236,76,11,171,86,246,216,120,133,37,98,194,63,159,99,195,62,158,217,121,132,36,10,170,87,247,176,16,237,77,8,168,85,245,178,18,239,79,97,193,60,156,219,123,134,38,218,122,135,39,96,192,61,157,179,19,238,78,9,169,84,244,0,161,95,254,190,31,225,64,97,192,62,159,223,126,128,33,194,99,157,60,124,221,35,130,163,2,252,93,29,188,66,227,153,56,198,103,39,134,120,217,248,89,167,6,70,231,25,184,91,250,4,165,229,68,186,27,58,155,101,196,132,37,219,122,47,142,112,209,145,48,206,111,78,239,17,176,240,81,175,14,237,76,178,19,83,242,12,173,140,45,211,114,50,147,109,204,182,23,233,72,8,169,87,246,215,118,136,41,105,200,54,151,116,213,43,138,202,107,149,52,21,180,74,235,171,10,244,85,94,255,1,160,224,65,191,30,63,158,96,193,129,32,222,127,156,61,195,98,34,131,125,220,253,92,162,3,67,226,28,189,199,102,152,57,121,216,38,135,166,7,249,88,24,185,71,230,5,164,90,251,187,26,228,69,100,197,59,154,218,123,133,36,113,208,46,143,207,110,144,49,16,177,79,238,174,15,241,80,179,18,236,77,13,172,82,243,210,115,141,44,108,205,51,146,232,73,183,22,86,247,9,168,137,40,214,119,55,150,104,201,42,139,117,212,148,53,203,106,75,234,20,181,245,84,170,11,0,162,89,251,178,16,235,73,121,219,32,130,203,105,146,48,242,80,171,9,64,226,25,187,139,41,210,112,57,155,96,194,249,91,160,2,75,233,18,176,128,34,217,123,50,144,107,201,11,169,82,240,185,27,224,66,114,208,43,137,192,98,153,59,239,77,182,20,93,255,4,166,150,52,207,109,36,134,125,223,29,191,68,230,175,13,246,84,100,198,61,159,214,116,143,45,22,180,79,237,164,6,253,95,111,205,54,148,221,127,132,38,228,70,189,31,86,244,15,173,157,63,196,102,47,141,118,212,195,97,154,56,113,211,40,138,186,24,227,65,8,170,81,243,49,147,104,202,131,33,218,120,72,234,17,179,250,88,163,1,58,152,99,193,136,42,209,115,67,225,26,184,241,83,168,10,200,106,145,51,122,216,35,129,177,19,232,74,3,161,90,248,44,142,117,215,158,60,199,101,85,247,12,174,231,69,190,28,222,124,135,37,108,206,53,151,167,5,254,92,21,183,76,238,213,119,140,46,103,197,62,156,172,14,245,87,30,188,71,229,39,133,126,220,149,55,204,110,94,252,7,165,236,78,181,23,0,163,91,248,182,21,237,78,113,210,42,137,199,100,156,63,226,65,185,26,84,247,15,172,147,48,200,107,37,134,126,221,217,122,130,33,111,204,52,151,168,11,243,80,30,189,69,230,59,152,96,195,141,46,214,117,74,233,17,178,252,95,167,4,175,12,244,87,25,186,66,225,222,125,133,38,104,203,51,144,77,238,22,181,251,88,160,3,60,159,103,196,138,41,209,114,118,213,45,142,192,99,155,56,7,164,92,255,177,18,234,73,148,55,207,108,34,129,121,218,229,70,190,29,83,240,8,171,67,224,24,187,245,86,174,13,50,145,105,202,132,39,223,124,161,2,250,89,23,180,76,239,208,115,139,40,102,197,61,158,154,57,193,98,44,143,119,212,235,72,176,19,93,254,6,165,120,219,35,128,206,109,149,54,9,170,82,241,191,28,228,71,236,79,183,20,90,249,1,162,157,62,198,101,43,136,112,211,14,173,85,246,184,27,227,64,127,220,36,135,201,106,146,49,53,150,110,205,131,32,216,123,68,231,31,188,242,81,169,10,215,116,140,47,97,194,58,153,166,5,253,94,16,179,75,232,0,164,85,241,170,14,255,91,73,237,28,184,227,71,182,18,146,54,199,99,56,156,109,201,219,127,142,42,113,213,36,128,57,157,108,200,147,55,198,98,112,212,37,129,218,126,143,43,171,15,254,90,1,165,84,240,226,70,183,19,72,236,29,185,114,214,39,131,216,124,141,41,59,159,110,202,145,53,196,96,224,68,181,17,74,238,31,187,169,13,252,88,3,167,86,242,75,239,30,186,225,69,180,16,2,166,87,243,168,12,253,89,217,125,140,40,115,215,38,130,144,52,197,97,58,158,111,203,228,64,177,21,78,234,27,191,173,9,248,92,7,163,82,246,118,210,35,135,220,120,137,45,63,155,106,206,149,49,192,100,221,121,136,44,119,211,34,134,148,48,193,101,62,154,107,207,79,235,26,190,229,65,176,20,6,162,83,247,172,8,249,93,150,50,195,103,60,152,105,205,223,123,138,46,117,209,32,132,4,160,81,245,174,10,251,95,77,233,24,188,231,67,178,22,175,11,250,94,5,161,80,244,230,66,179,23,76,232,25,189,61,153,104,204,151,51,194,102,116,208,33,133,222,122,139,47,0,165,87,242,174,11,249,92,65,228,22,179,239,74,184,29,130,39,213,112,44,137,123,222,195,102,148,49,109,200,58,159,25,188,78,235,183,18,224,69,88,253,15,170,246,83,161,4,155,62,204,105,53,144,98,199,218,127,141,40,116,209,35,134,50,151,101,192,156,57,203,110,115,214,36,129,221,120,138,47,176,21,231,66,30,187,73,236,241,84,166,3,95,250,8,173,43,142,124,217,133,32,210,119,106,207,61,152,196,97,147,54,169,12,254,91,7,162,80,245,232,77,191,26,70,227,17,180,100,193,51,150,202,111,157,56,37,128,114,215,139,46,220,121,230,67,177,20,72,237,31,186,167,2,240,85,9,172,94,251,125,216,42,143,211,118,132,33,60,153,107,206,146,55,197,96,255,90,168,13,81,244,6,163,190,27,233,76,16,181,71,226,86,243,1,164,248,93,175,10,23,178,64,229,185,28,238,75,212,113,131,38,122,223,45,136,149,48,194,103,59,158,108,201,79,234,24,189,225,68,182,19,14,171,89,252,160,5,247,82,205,104,154,63,99,198,52,145,140,41,219,126,34,135,117,208,0,166,81,247,162,4,243,85,89,255,8,174,251,93,170,12,178,20,227,69,16,182,65,231,235,77,186,28,73,239,24,190,121,223,40,142,219,125,138,44,32,134,113,215,130,36,211,117,203,109,154,60,105,207,56,158,146,52,195,101,48,150,97,199,242,84,163,5,80,246,1,167,171,13,250,92,9,175,88,254,64,230,17,183,226,68,179,21,25,191,72,238,187,29,234,76,139,45,218,124,41,143,120,222,210,116,131,37,112,214,33,135,57,159,104,206,155,61,202,108,96,198,49,151,194,100,147,53,249,95,168,14,91,253,10,172,160,6,241,87,2,164,83,245,75,237,26,188,233,79,184,30,18,180,67,229,176,22,225,71,128,38,209,119,34,132,115,213,217,127,136,46,123,221,42,140,50,148,99,197,144,54,193,103,107,205,58,156,201,111,152,62,11,173,90,252,169,15,248,94,82,244,3,165,240,86,161,7,185,31,232,78,27,189,74,236,224,70,177,23,66,228,19,181,114,212,35,133,208,118,129,39,43,141,122,220,137,47,216,126,192,102,145,55,98,196,51,149,153,63,200,110,59,157,106,204,0,167,83,244,166,1,245,82,81,246,2,165,247,80,164,3,162,5,241,86,4,163,87,240,243,84,160,7,85,242,6,161,89,254,10,173,255,88,172,11,8,175,91,252,174,9,253,90,251,92,168,15,93,250,14,169,170,13,249,94,12,171,95,248,178,21,225,70,20,179,71,224,227,68,176,23,69,226,22,177,16,183,67,228,182,17,229,66,65,230,18,181,231,64,180,19,235,76,184,31,77,234,30,185,186,29,233,78,28,187,79,232,73,238,26,189,239,72,188,27,24,191,75,236,190,25,237,74,121,222,42,141,223,120,140,43,40,143,123,220,142,41,221,122,219,124,136,47,125,218,46,137,138,45,217,126,44,139,127,216,32,135,115,212,134,33,213,114,113,214,34,133,215,112,132,35,130,37,209,118,36,131,119,208,211,116,128,39,117,210,38,129,203,108,152,63,109,202,62,153,154,61,201,110,60,155,111,200,105,206,58,157,207,104,156,59,56,159,107,204,158,57,205,106,146,53,193,102,52,147,103,192,195,100,144,55,101,194,54,145,48,151,99,196,150,49,197,98,97,198,50,149,199,96,148,51,0,168,77,229,154,50,215,127,41,129,100,204,179,27,254,86,82,250,31,183,200,96,133,45,123,211,54,158,225,73,172,4,164,12,233,65,62,150,115,219,141,37,192,104,23,191,90,242,246,94,187,19,108,196,33,137,223,119,146,58,69,237,8,160,85,253,24,176,207,103,130,42,124,212,49,153,230,78,171,3,7,175,74,226,157,53,208,120,46,134,99,203,180,28,249,81,241,89,188,20,107,195,38,142,216,112,149,61,66,234,15,167,163,11,238,70,57,145,116,220,138,34,199,111,16,184,93,245,170,2,231,79,48,152,125,213,131,43,206,102,25,177,84,252,248,80,181,29,98,202,47,135,209,121,156,52,75,227,6,174,14,166,67,235,148,60,217,113,39,143,106,194,189,21,240,88,92,244,17,185,198,110,139,35,117,221,56,144,239,71,162,10,255,87,178,26,101,205,40,128,214,126,155,51,76,228,1,169,173,5,224,72,55,159,122,210,132,44,201,97,30,182,83,251,91,243,22,190,193,105,140,36,114,218,63,151,232,64,165,13,9,161,68,236,147,59,222,118,32,136,109,197,186,18,247,95,0,169,79,230,158,55,209,120,33,136,110,199,191,22,240,89,66,235,13,164,220,117,147,58,99,202,44,133,253,84,178,27,132,45,203,98,26,179,85,252,165,12,234,67,59,146,116,221,198,111,137,32,88,241,23,190,231,78,168,1,121,208,54,159,21,188,90,243,139,34,196,109,52,157,123,210,170,3,229,76,87,254,24,177,201,96,134,47,118,223,57,144,232,65,167,14,145,56,222,119,15,166,64,233,176,25,255,86,46,135,97,200,211,122,156,53,77,228,2,171,242,91,189,20,108,197,35,138,42,131,101,204,180,29,251,82,11,162,68,237,149,60,218,115,104,193,39,142,246,95,185,16,73,224,6,175,215,126,152,49,174,7,225,72,48,153,127,214,143,38,192,105,17,184,94,247,236,69,163,10,114,219,61,148,205,100,130,43,83,250,28,181,63,150,112,217,161,8,238,71,30,183,81,248,128,41,207,102,125,212,50,155,227,74,172,5,92,245,19,186,194,107,141,36,187,18,244,93,37,140,106,195,154,51,213,124,4,173,75,226,249,80,182,31,103,206,40,129,216,113,151,62,70,239,9,160,0,170,73,227,146,56,219,113,57,147,112,218,171,1,226,72,114,216,59,145,224,74,169,3,75,225,2,168,217,115,144,58,228,78,173,7,118,220,63,149,221,119,148,62,79,229,6,172,150,60,223,117,4,174,77,231,175,5,230,76,61,151,116,222,213,127,156,54,71,237,14,164,236,70,165,15,126,212,55,157,167,13,238,68,53,159,124,214,158,52,215,125,12,166,69,239,49,155,120,210,163,9,234,64,8,162,65,235,154,48,211,121,67,233,10,160,209,123,152,50,122,208,51,153,232,66,161,11,183,29,254,84,37,143,108,198,142,36,199,109,28,182,85,255,197,111,140,38,87,253,30,180,252,86,181,31,110,196,39,141,83,249,26,176,193,107,136,34,106,192,35,137,248,82,177,27,33,139,104,194,179,25,250,80,24,178,81,251,138,32,195,105,98,200,43,129,240,90,185,19,91,241,18,184,201,99,128,42,16,186,89,243,130,40,203,97,41,131,96,202,187,17,242,88,134,44,207,101,20,190,93,247,191,21,246,92,45,135,100,206,244,94,189,23,102,204,47,133,205,103,132,46,95,245,22,188,0,171,75,224,150,61,221,118,49,154,122,209,167,12,236,71,98,201,41,130,244,95,191,20,83,248,24,179,197,110,142,37,196,111,143,36,82,249,25,178,245,94,190,21,99,200,40,131,166,13,237,70,48,155,123,208,151,60,220,119,1,170,74,225,149,62,222,117,3,168,72,227,164,15,239,68,50,153,121,210,247,92,188,23,97,202,42,129,198,109,141,38,80,251,27,176,81,250,26,177,199,108,140,39,96,203,43,128,246,93,189,22,51,152,120,211,165,14,238,69,2,169,73,226,148,63,223,116,55,156,124,215,161,10,234,65,6,173,77,230,144,59,219,112,85,254,30,181,195,104,136,35,100,207,47,132,242,89,185,18,243,88,184,19,101,206,46,133,194,105,137,34,84,255,31,180,145,58,218,113,7,172,76,231,160,11,235,64,54,157,125,214,162,9,233,66,52,159,127,212,147,56,216,115,5,174,78,229,192,107,139,32,86,253,29,182,241,90,186,17,103,204,44,135,102,205,45,134,240,91,187,16,87,252,28,183,193,106,138,33,4,175,79,228,146,57,217,114,53,158,126,213,163,8,232,67,0,172,69,233,138,38,207,99,9,165,76,224,131,47,198,106,18,190,87,251,152,52,221,113,27,183,94,242,145,61,212,120,36,136,97,205,174,2,235,71,45,129,104,196,167,11,226,78,54,154,115,223,188,16,249,85,63,147,122,214,181,25,240,92,72,228,13,161,194,110,135,43,65,237,4,168,203,103,142,34,90,246,31,179,208,124,149,57,83,255,22,186,217,117,156,48,108,192,41,133,230,74,163,15,101,201,32,140,239,67,170,6,126,210,59,151,244,88,177,29,119,219,50,158,253,81,184,20,144,60,213,121,26,182,95,243,153,53,220,112,19,191,86,250,130,46,199,107,8,164,77,225,139,39,206,98,1,173,68,232,180,24,241,93,62,146,123,215,189,17,248,84,55,155,114,222,166,10,227,79,44,128,105,197,175,3,234,70,37,137,96,204,216,116,157,49,82,254,23,187,209,125,148,56,91,247,30,178,202,102,143,35,64,236,5,169,195,111,134,42,73,229,12,160,252,80,185,21,118,218,51,159,245,89,176,28,127,211,58,150,238,66,171,7,100,200,33,141,231,75,162,14,109,193,40,132,0,173,71,234,142,35,201,100,1,172,70,235,143,34,200,101,2,175,69,232,140,33,203,102,3,174,68,233,141,32,202,103,4,169,67,238,138,39,205,96,5,168,66,239,139,38,204,97,6,171,65,236,136,37,207,98,7,170,64,237,137,36,206,99,8,165,79,226,134,43,193,108,9,164,78,227,135,42,192,109,10,167,77,224,132,41,195,110,11,166,76,225,133,40,194,111,12,161,75,230,130,47,197,104,13,160,74,231,131,46,196,105,14,163,73,228,128,45,199,106,15,162,72,229,129,44,198,107,16,189,87,250,158,51,217,116,17,188,86,251,159,50,216,117,18,191,85,248,156,49,219,118,19,190,84,249,157,48,218,119,20,185,83,254,154,55,221,112,21,184,82,255,155,54,220,113,22,187,81,252,152,53,223,114,23,186,80,253,153,52,222,115,24,181,95,242,150,59,209,124,25,180,94,243,151,58,208,125,26,183,93,240,148,57,211,126,27,182,92,241,149,56,210,127,28,177,91,246,146,63,213,120,29,176,90,247,147,62,212,121,30,179,89,244,144,61,215,122,31,178,88,245,145,60,214,123,0,174,65,239,130,44,195,109,25,183,88,246,155,53,218,116,50,156,115,221,176,30,241,95,43,133,106,196,169,7,232,70,100,202,37,139,230,72,167,9,125,211,60,146,255,81,190,16,86,248,23,185,212,122,149,59,79,225,14,160,205,99,140,34,200,102,137,39,74,228,11,165,209,127,144,62,83,253,18,188,250,84,187,21,120,214,57,151,227,77,162,12,97,207,32,142,172,2,237,67,46,128,111,193,181,27,244,90,55,153,118,216,158,48,223,113,28,178,93,243,135,41,198,104,5,171,68,234,141,35,204,98,15,161,78,224,148,58,213,123,22,184,87,249,191,17,254,80,61,147,124,210,166,8,231,73,36,138,101,203,233,71,168,6,107,197,42,132,240,94,177,31,114,220,51,157,219,117,154,52,89,247,24,182,194,108,131,45,64,238,1,175,69,235,4,170,199,105,134,40,92,242,29,179,222,112,159,49,119,217,54,152,245,91,180,26,110,192,47,129,236,66,173,3,33,143,96,206,163,13,226,76,56,150,121,215,186,20,251,85,19,189,82,252,145,63,208,126,10,164,75,229,136,38,201,103,0,175,67,236,134,41,197,106,17,190,82,253,151,56,212,123,34,141,97,206,164,11,231,72,51,156,112,223,181,26,246,89,68,235,7,168,194,109,129,46,85,250,22,185,211,124,144,63,102,201,37,138,224,79,163,12,119,216,52,155,241,94,178,29,136,39,203,100,14,161,77,226,153,54,218,117,31,176,92,243,170,5,233,70,44,131,111,192,187,20,248,87,61,146,126,209,204,99,143,32,74,229,9,166,221,114,158,49,91,244,24,183,238,65,173,2,104,199,43,132,255,80,188,19,121,214,58,149,13,162,78,225,139,36,200,103,28,179,95,240,154,53,217,118,47,128,108,195,169,6,234,69,62,145,125,210,184,23,251,84,73,230,10,165,207,96,140,35,88,247,27,180,222,113,157,50,107,196,40,135,237,66,174,1,122,213,57,150,252,83,191,16,133,42,198,105,3,172,64,239,148,59,215,120,18,189,81,254,167,8,228,75,33,142,98,205,182,25,245,90,48,159,115,220,193,110,130,45,71,232,4,171,208,127,147,60,86,249,21,186,227,76,160,15,101,202,38,137,242,93,177,30,116,219,55,152,0,176,125,205,250,74,135,55,233,89,148,36,19,163,110,222,207,127,178,2,53,133,72,248,38,150,91,235,220,108,161,17,131,51,254,78,121,201,4,180,106,218,23,167,144,32,237,93,76,252,49,129,182,6,203,123,165,21,216,104,95,239,34,146,27,171,102,214,225,81,156,44,242,66,143,63,8,184,117,197,212,100,169,25,46,158,83,227,61,141,64,240,199,119,186,10,152,40,229,85,98,210,31,175,113,193,12,188,139,59,246,70,87,231,42,154,173,29,208,96,190,14,195,115,68,244,57,137,54,134,75,251,204,124,177,1,223,111,162,18,37,149,88,232,249,73,132,52,3,179,126,206,16,160,109,221,234,90,151,39,181,5,200,120,79,255,50,130,92,236,33,145,166,22,219,107,122,202,7,183,128,48,253,77,147,35,238,94,105,217,20,164,45,157,80,224,215,103,170,26,196,116,185,9,62,142,67,243,226,82,159,47,24,168,101,213,11,187,118,198,241,65,140,60,174,30,211,99,84,228,41,153,71,247,58,138,189,13,192,112,97,209,28,172,155,43,230,86,136,56,245,69,114,194,15,191,0,177,127,206,254,79,129,48,225,80,158,47,31,174,96,209,223,110,160,17,33,144,94,239,62,143,65,240,192,113,191,14,163,18,220,109,93,236,34,147,66,243,61,140,188,13,195,114,124,205,3,178,130,51,253,76,157,44,226,83,99,210,28,173,91,234,36,149,165,20,218,107,186,11,197,116,68,245,59,138,132,53,251,74,122,203,5,180,101,212,26,171,155,42,228,85,248,73,135,54,6,183,121,200,25,168,102,215,231,86,152,41,39,150,88,233,217,104,166,23,198,119,185,8,56,137,71,246,182,7,201,120,72,249,55,134,87,230,40,153,169,24,214,103,105,216,22,167,151,38,232,89,136,57,247,70,118,199,9,184,21,164,106,219,235,90,148,37,244,69,139,58,10,187,117,196,202,123,181,4,52,133,75,250,43,154,84,229,213,100,170,27,237,92,146,35,19,162,108,221,12,189,115,194,242,67,141,60,50,131,77,252,204,125,179,2,211,98,172,29,45,156,82,227,78,255,49,128,176,1,207,126,175,30,208,97,81,224,46,159,145,32,238,95,111,222,16,161,112,193,15,190,142,63,241,64,0,178,121,203,242,64,139,57,249,75,128,50,11,185,114,192,239,93,150,36,29,175,100,214,22,164,111,221,228,86,157,47,195,113,186,8,49,131,72,250,58,136,67,241,200,122,177,3,44,158,85,231,222,108,167,21,213,103,172,30,39,149,94,236,155,41,226,80,105,219,16,162,98,208,27,169,144,34,233,91,116,198,13,191,134,52,255,77,141,63,244,70,127,205,6,180,88,234,33,147,170,24,211,97,161,19,216,106,83,225,42,152,183,5,206,124,69,247,60,142,78,252,55,133,188,14,197,119,43,153,82,224,217,107,160,18,210,96,171,25,32,146,89,235,196,118,189,15,54,132,79,253,61,143,68,246,207,125,182,4,232,90,145,35,26,168,99,209,17,163,104,218,227,81,154,40,7,181,126,204,245,71,140,62,254,76,135,53,12,190,117,199,176,2,201,123,66,240,59,137,73,251,48,130,187,9,194,112,95,237,38,148,173,31,212,102,166,20,223,109,84,230,45,159,115,193,10,184,129,51,248,74,138,56,243,65,120,202,1,179,156,46,229,87,110,220,23,165,101,215,28,174,151,37,238,92,0,179,123,200,246,69,141,62,241,66,138,57,7,180,124,207,255,76,132,55,9,186,114,193,14,189,117,198,248,75,131,48,227,80,152,43,21,166,110,221,18,161,105,218,228,87,159,44,28,175,103,212,234,89,145,34,237,94,150,37,27,168,96,211,219,104,160,19,45,158,86,229,42,153,81,226,220,111,167,20,36,151,95,236,210,97,169,26,213,102,174,29,35,144,88,235,56,139,67,240,206,125,181,6,201,122,178,1,63,140,68,247,199,116,188,15,49,130,74,249,54,133,77,254,192,115,187,8,171,24,208,99,93,238,38,149,90,233,33,146,172,31,215,100,84,231,47,156,162,17,217,106,165,22,222,109,83,224,40,155,72,251,51,128,190,13,197,118,185,10,194,113,79,252,52,135,183,4,204,127,65,242,58,137,70,245,61,142,176,3,203,120,112,195,11,184,134,53,253,78,129,50,250,73,119,196,12,191,143,60,244,71,121,202,2,177,126,205,5,182,136,59,243,64,147,32,232,91,101,214,30,173,98,209,25,170,148,39,239,92,108,223,23,164,154,41,225,82,157,46,230,85,107,216,16,163,0,180,117,193,234,94,159,43,201,125,188,8,35,151,86,226,143,59,250,78,101,209,16,164,70,242,51,135,172,24,217,109,3,183,118,194,233,93,156,40,202,126,191,11,32,148,85,225,140,56,249,77,102,210,19,167,69,241,48,132,175,27,218,110,6,178,115,199,236,88,153,45,207,123,186,14,37,145,80,228,137,61,252,72,99,215,22,162,64,244,53,129,170,30,223,107,5,177,112,196,239,91,154,46,204,120,185,13,38,146,83,231,138,62,255,75,96,212,21,161,67,247,54,130,169,29,220,104,12,184,121,205,230,82,147,39,197,113,176,4,47,155,90,238,131,55,246,66,105,221,28,168,74,254,63,139,160,20,213,97,15,187,122,206,229,81,144,36,198,114,179,7,44,152,89,237,128,52,245,65,106,222,31,171,73,253,60,136,163,23,214,98,10,190,127,203,224,84,149,33,195,119,182,2,41,157,92,232,133,49,240,68,111,219,26,174,76,248,57,141,166,18,211,103,9,189,124,200,227,87,150,34,192,116,181,1,42,158,95,235,134,50,243,71,108,216,25,173,79,251,58,142,165,17,208,100,0,181,119,194,238,91,153,44,193,116,182,3,47,154,88,237,159,42,232,93,113,196,6,179,94,235,41,156,176,5,199,114,35,150,84,225,205,120,186,15,226,87,149,32,12,185,123,206,188,9,203,126,82,231,37,144,125,200,10,191,147,38,228,81,70,243,49,132,168,29,223,106,135,50,240,69,105,220,30,171,217,108,174,27,55,130,64,245,24,173,111,218,246,67,129,52,101,208,18,167,139,62,252,73,164,17,211,102,74,255,61,136,250,79,141,56,20,161,99,214,59,142,76,249,213,96,162,23,140,57,251,78,98,215,21,160,77,248,58,143,163,22,212,97,19,166,100,209,253,72,138,63,210,103,165,16,60,137,75,254,175,26,216,109,65,244,54,131,110,219,25,172,128,53,247,66,48,133,71,242,222,107,169,28,241,68,134,51,31,170,104,221,202,127,189,8,36,145,83,230,11,190,124,201,229,80,146,39,85,224,34,151,187,14,204,121,148,33,227,86,122,207,13,184,233,92,158,43,7,178,112,197,40,157,95,234,198,115,177,4,118,195,1,180,152,45,239,90,183,2,192,117,89,236,46,155,0,182,113,199,226,84,147,37,217,111,168,30,59,141,74,252,175,25,222,104,77,251,60,138,118,192,7,177,148,34,229,83,67,245,50,132,161,23,208,102,154,44,235,93,120,206,9,191,236,90,157,43,14,184,127,201,53,131,68,242,215,97,166,16,134,48,247,65,100,210,21,163,95,233,46,152,189,11,204,122,41,159,88,238,203,125,186,12,240,70,129,55,18,164,99,213,197,115,180,2,39,145,86,224,28,170,109,219,254,72,143,57,106,220,27,173,136,62,249,79,179,5,194,116,81,231,32,150,17,167,96,214,243,69,130,52,200,126,185,15,42,156,91,237,190,8,207,121,92,234,45,155,103,209,22,160,133,51,244,66,82,228,35,149,176,6,193,119,139,61,250,76,105,223,24,174,253,75,140,58,31,169,110,216,36,146,85,227,198,112,183,1,151,33,230,80,117,195,4,178,78,248,63,137,172,26,221,107,56,142,73,255,218,108,171,29,225,87,144,38,3,181,114,196,212,98,165,19,54,128,71,241,13,187,124,202,239,89,158,40,123,205,10,188,153,47,232,94,162,20,211,101,64,246,49,135,0,183,115,196,230,81,149,34,209,102,162,21,55,128,68,243,191,8,204,123,89,238,42,157,110,217,29,170,136,63,251,76,99,212,16,167,133,50,246,65,178,5,193,118,84,227,39,144,220,107,175,24,58,141,73,254,13,186,126,201,235,92,152,47,198,113,181,2,32,151,83,228,23,160,100,211,241,70,130,53,121,206,10,189,159,40,236,91,168,31,219,108,78,249,61,138,165,18,214,97,67,244,48,135,116,195,7,176,146,37,225,86,26,173,105,222,252,75,143,56,203,124,184,15,45,154,94,233,145,38,226,85,119,192,4,179,64,247,51,132,166,17,213,98,46,153,93,234,200,127,187,12,255,72,140,59,25,174,106,221,242,69,129,54,20,163,103,208,35,148,80,231,197,114,182,1,77,250,62,137,171,28,216,111,156,43,239,88,122,205,9,190,87,224,36,147,177,6,194,117,134,49,245,66,96,215,19,164,232,95,155,44,14,185,125,202,57,142,74,253,223,104,172,27,52,131,71,240,210,101,161,22,229,82,150,33,3,180,112,199,139,60,248,79,109,218,30,169,90,237,41,158,188,11,207,120,0,184,109,213,218,98,183,15,169,17,196,124,115,203,30,166,79,247,34,154,149,45,248,64,230,94,139,51,60,132,81,233,158,38,243,75,68,252,41,145,55,143,90,226,237,85,128,56,209,105,188,4,11,179,102,222,120,192,21,173,162,26,207,119,33,153,76,244,251,67,150,46,136,48,229,93,82,234,63,135,110,214,3,187,180,12,217,97,199,127,170,18,29,165,112,200,191,7,210,106,101,221,8,176,22,174,123,195,204,116,161,25,240,72,157,37,42,146,71,255,89,225,52,140,131,59,238,86,66,250,47,151,152,32,245,77,235,83,134,62,49,137,92,228,13,181,96,216,215,111,186,2,164,28,201,113,126,198,19,171,220,100,177,9,6,190,107,211,117,205,24,160,175,23,194,122,147,43,254,70,73,241,36,156,58,130,87,239,224,88,141,53,99,219,14,182,185,1,212,108,202,114,167,31,16,168,125,197,44,148,65,249,246,78,155,35,133,61,232,80,95,231,50,138,253,69,144,40,39,159,74,242,84,236,57,129,142,54,227,91,178,10,223,103,104,208,5,189,27,163,118,206,193,121,172,20,0,185,111,214,222,103,177,8,161,24,206,119,127,198,16,169,95,230,48,137,129,56,238,87,254,71,145,40,32,153,79,246,190,7,209,104,96,217,15,182,31,166,112,201,193,120,174,23,225,88,142,55,63,134,80,233,64,249,47,150,158,39,241,72,97,216,14,183,191,6,208,105,192,121,175,22,30,167,113,200,62,135,81,232,224,89,143,54,159,38,240,73,65,248,46,151,223,102,176,9,1,184,110,215,126,199,17,168,160,25,207,118,128,57,239,86,94,231,49,136,33,152,78,247,255,70,144,41,194,123,173,20,28,165,115,202,99,218,12,181,189,4,210,107,157,36,242,75,67,250,44,149,60,133,83,234,226,91,141,52,124,197,19,170,162,27,205,116,221,100,178,11,3,186,108,213,35,154,76,245,253,68,146,43,130,59,237,84,92,229,51,138,163,26,204,117,125,196,18,171,2,187,109,212,220,101,179,10,252,69,147,42,34,155,77,244,93,228,50,139,131,58,236,85,29,164,114,203,195,122,172,21,188,5,211,106,98,219,13,180,66,251,45,148,156,37,243,74,227,90,140,53,61,132,82,235,0,186,105,211,210,104,187,1,185,3,208,106,107,209,2,184,111,213,6,188,189,7,212,110,214,108,191,5,4,190,109,215,222,100,183,13,12,182,101,223,103,221,14,180,181,15,220,102,177,11,216,98,99,217,10,176,8,178,97,219,218,96,179,9,161,27,200,114,115,201,26,160,24,162,113,203,202,112,163,25,206,116,167,29,28,166,117,207,119,205,30,164,165,31,204,118,127,197,22,172,173,23,196,126,198,124,175,21,20,174,125,199,16,170,121,195,194,120,171,17,169,19,192,122,123,193,18,168,95,229,54,140,141,55,228,94,230,92,143,53,52,142,93,231,48,138,89,227,226,88,139,49,137,51,224,90,91,225,50,136,129,59,232,82,83,233,58,128,56,130,81,235,234,80,131,57,238,84,135,61,60,134,85,239,87,237,62,132,133,63,236,86,254,68,151,45,44,150,69,255,71,253,46,148,149,47,252,70,145,43,248,66,67,249,42,144,40,146,65,251,250,64,147,41,32,154,73,243,242,72,155,33,153,35,240,74,75,241,34,152,79,245,38,156,157,39,244,78,246,76,159,37,36,158,77,247,0,187,107,208,214,109,189,6,177,10,218,97,103,220,12,183,127,196,20,175,169,18,194,121,206,117,165,30,24,163,115,200,254,69,149,46,40,147,67,248,79,244,36,159,153,34,242,73,129,58,234,81,87,236,60,135,48,139,91,224,230,93,141,54,225,90,138,49,55,140,92,231,80,235,59,128,134,61,237,86,158,37,245,78,72,243,35,152,47,148,68,255,249,66,146,41,31,164,116,207,201,114,162,25,174,21,197,126,120,195,19,168,96,219,11,176,182,13,221,102,209,106,186,1,7,188,108,215,223,100,180,15,9,178,98,217,110,213,5,190,184,3,211,104,160,27,203,112,118,205,29,166,17,170,122,193,199,124,172,23,33,154,74,241,247,76,156,39,144,43,251,64,70,253,45,150,94,229,53,142,136,51,227,88,239,84,132,63,57,130,82,233,62,133,85,238,232,83,131,56,143,52,228,95,89,226,50,137,65,250,42,145,151,44,252,71,240,75,155,32,38,157,77,246,192,123,171,16,22,173,125,198,113,202,26,161,167,28,204,119,191,4,212,111,105,210,2,185,14,181,101,222,216,99,179,8,0,188,101,217,202,118,175,19,137,53,236,80,67,255,38,154,15,179,106,214,197,121,160,28,134,58,227,95,76,240,41,149,30,162,123,199,212,104,177,13,151,43,242,78,93,225,56,132,17,173,116,200,219,103,190,2,152,36,253,65,82,238,55,139,60,128,89,229,246,74,147,47,181,9,208,108,127,195,26,166,51,143,86,234,249,69,156,32,186,6,223,99,112,204,21,169,34,158,71,251,232,84,141,49,171,23,206,114,97,221,4,184,45,145,72,244,231,91,130,62,164,24,193,125,110,210,11,183,120,196,29,161,178,14,215,107,241,77,148,40,59,135,94,226,119,203,18,174,189,1,216,100,254,66,155,39,52,136,81,237,102,218,3,191,172,16,201,117,239,83,138,54,37,153,64,252,105,213,12,176,163,31,198,122,224,92,133,57,42,150,79,243,68,248,33,157,142,50,235,87,205,113,168,20,7,187,98,222,75,247,46,146,129,61,228,88,194,126,167,27,8,180,109,209,90,230,63,131,144,44,245,73,211,111,182,10,25,165,124,192,85,233,48,140,159,35,250,70,220,96,185,5,22,170,115,207,0,189,103,218,206,115,169,20,129,60,230,91,79,242,40,149,31,162,120,197,209,108,182,11,158,35,249,68,80,237,55,138,62,131,89,228,240,77,151,42,191,2,216,101,113,204,22,171,33,156,70,251,239,82,136,53,160,29,199,122,110,211,9,180,124,193,27,166,178,15,213,104,253,64,154,39,51,142,84,233,99,222,4,185,173,16,202,119,226,95,133,56,44,145,75,246,66,255,37,152,140,49,235,86,195,126,164,25,13,176,106,215,93,224,58,135,147,46,244,73,220,97,187,6,18,175,117,200,248,69,159,34,54,139,81,236,121,196,30,163,183,10,208,109,231,90,128,61,41,148,78,243,102,219,1,188,168,21,207,114,198,123,161,28,8,181,111,210,71,250,32,157,137,52,238,83,217,100,190,3,23,170,112,205,88,229,63,130,150,43,241,76,132,57,227,94,74,247,45,144,5,184,98,223,203,118,172,17,155,38,252,65,85,232,50,143,26,167,125,192,212,105,179,14,186,7,221,96,116,201,19,174,59,134,92,225,245,72,146,47,165,24,194,127,107,214,12,177,36,153,67,254,234,87,141,48,0,190,97,223,194,124,163,29,153,39,248,70,91,229,58,132,47,145,78,240,237,83,140,50,182,8,215,105,116,202,21,171,94,224,63,129,156,34,253,67,199,121,166,24,5,187,100,218,113,207,16,174,179,13,210,108,232,86,137,55,42,148,75,245,188,2,221,99,126,192,31,161,37,155,68,250,231,89,134,56,147,45,242,76,81,239,48,142,10,180,107,213,200,118,169,23,226,92,131,61,32,158,65,255,123,197,26,164,185,7,216,102,205,115,172,18,15,177,110,208,84,234,53,139,150,40,247,73,101,219,4,186,167,25,198,120,252,66,157,35,62,128,95,225,74,244,43,149,136,54,233,87,211,109,178,12,17,175,112,206,59,133,90,228,249,71,152,38,162,28,195,125,96,222,1,191,20,170,117,203,214,104,183,9,141,51,236,82,79,241,46,144,217,103,184,6,27,165,122,196,64,254,33,159,130,60,227,93,246,72,151,41,52,138,85,235,111,209,14,176,173,19,204,114,135,57,230,88,69,251,36,154,30,160,127,193,220,98,189,3,168,22,201,119,106,212,11,181,49,143,80,238,243,77,146,44,0,191,99,220,198,121,165,26,145,46,242,77,87,232,52,139,63,128,92,227,249,70,154,37,174,17,205,114,104,215,11,180,126,193,29,162,184,7,219,100,239,80,140,51,41,150,74,245,65,254,34,157,135,56,228,91,208,111,179,12,22,169,117,202,252,67,159,32,58,133,89,230,109,210,14,177,171,20,200,119,195,124,160,31,5,186,102,217,82,237,49,142,148,43,247,72,130,61,225,94,68,251,39,152,19,172,112,207,213,106,182,9,189,2,222,97,123,196,24,167,44,147,79,240,234,85,137,54,229,90,134,57,35,156,64,255,116,203,23,168,178,13,209,110,218,101,185,6,28,163,127,192,75,244,40,151,141,50,238,81,155,36,248,71,93,226,62,129,10,181,105,214,204,115,175,16,164,27,199,120,98,221,1,190,53,138,86,233,243,76,144,47,25,166,122,197,223,96,188,3,136,55,235,84,78,241,45,146,38,153,69,250,224,95,131,60,183,8,212,107,113,206,18,173,103,216,4,187,161,30,194,125,246,73,149,42,48,143,83,236,88,231,59,132,158,33,253,66,201,118,170,21,15,176,108,211,0,192,157,93,39,231,186,122,78,142,211,19,105,169,244,52,156,92,1,193,187,123,38,230,210,18,79,143,245,53,104,168,37,229,184,120,2,194,159,95,107,171,246,54,76,140,209,17,185,121,36,228,158,94,3,195,247,55,106,170,208,16,77,141,74,138,215,23,109,173,240,48,4,196,153,89,35,227,190,126,214,22,75,139,241,49,108,172,152,88,5,197,191,127,34,226,111,175,242,50,72,136,213,21,33,225,188,124,6,198,155,91,243,51,110,174,212,20,73,137,189,125,32,224,154,90,7,199,148,84,9,201,179,115,46,238,218,26,71,135,253,61,96,160,8,200,149,85,47,239,178,114,70,134,219,27,97,161,252,60,177,113,44,236,150,86,11,203,255,63,98,162,216,24,69,133,45,237,176,112,10,202,151,87,99,163,254,62,68,132,217,25,222,30,67,131,249,57,100,164,144,80,13,205,183,119,42,234,66,130,223,31,101,165,248,56,12,204,145,81,43,235,182,118,251,59,102,166,220,28,65,129,181,117,40,232,146,82,15,207,103,167,250,58,64,128,221,29,41,233,180,116,14,206,147,83,0,193,159,94,35,226,188,125,70,135,217,24,101,164,250,59,140,77,19,210,175,110,48,241,202,11,85,148,233,40,118,183,5,196,154,91,38,231,185,120,67,130,220,29,96,161,255,62,137,72,22,215,170,107,53,244,207,14,80,145,236,45,115,178,10,203,149,84,41,232,182,119,76,141,211,18,111,174,240,49,134,71,25,216,165,100,58,251,192,1,95,158,227,34,124,189,15,206,144,81,44,237,179,114,73,136,214,23,106,171,245,52,131,66,28,221,160,97,63,254,197,4,90,155,230,39,121,184,20,213,139,74,55,246,168,105,82,147,205,12,113,176,238,47,152,89,7,198,187,122,36,229,222,31,65,128,253,60,98,163,17,208,142,79,50,243,173,108,87,150,200,9,116,181,235,42,157,92,2,195,190,127,33,224,219,26,68,133,248,57,103,166,30,223,129,64,61,252,162,99,88,153,199,6,123,186,228,37,146,83,13,204,177,112,46,239,212,21,75,138,247,54,104,169,27,218,132,69,56,249,167,102,93,156,194,3,126,191,225,32,151,86,8,201,180,117,43,234,209,16,78,143,242,51,109,172,0,194,153,91,47,237,182,116,94,156,199,5,113,179,232,42,188,126,37,231,147,81,10,200,226,32,123,185,205,15,84,150,101,167,252,62,74,136,211,17,59,249,162,96,20,214,141,79,217,27,64,130,246,52,111,173,135,69,30,220,168,106,49,243,202,8,83,145,229,39,124,190,148,86,13,207,187,121,34,224,118,180,239,45,89,155,192,2,40,234,177,115,7,197,158,92,175,109,54,244,128,66,25,219,241,51,104,170,222,28,71,133,19,209,138,72,60,254,165,103,77,143,212,22,98,160,251,57,137,75,16,210,166,100,63,253,215,21,78,140,248,58,97,163,53,247,172,110,26,216,131,65,107,169,242,48,68,134,221,31,236,46,117,183,195,1,90,152,178,112,43,233,157,95,4,198,80,146,201,11,127,189,230,36,14,204,151,85,33,227,184,122,67,129,218,24,108,174,245,55,29,223,132,70,50,240,171,105,255,61,102,164,208,18,73,139,161,99,56,250,142,76,23,213,38,228,191,125,9,203,144,82,120,186,225,35,87,149,206,12,154,88,3,193,181,119,44,238,196,6,93,159,235,41,114,176,0,195,155,88,43,232,176,115,86,149,205,14,125,190,230,37,172,111,55,244,135,68,28,223,250,57,97,162,209,18,74,137,69,134,222,29,110,173,245,54,19,208,136,75,56,251,163,96,233,42,114,177,194,1,89,154,191,124,36,231,148,87,15,204,138,73,17,210,161,98,58,249,220,31,71,132,247,52,108,175,38,229,189,126,13,206,150,85,112,179,235,40,91,152,192,3,207,12,84,151,228,39,127,188,153,90,2,193,178,113,41,234,99,160,248,59,72,139,211,16,53,246,174,109,30,221,133,70,9,202,146,81,34,225,185,122,95,156,196,7,116,183,239,44,165,102,62,253,142,77,21,214,243,48,104,171,216,27,67,128,76,143,215,20,103,164,252,63,26,217,129,66,49,242,170,105,224,35,123,184,203,8,80,147,182,117,45,238,157,94,6,197,131,64,24,219,168,107,51,240,213,22,78,141,254,61,101,166,47,236,180,119,4,199,159,92,121,186,226,33,82,145,201,10,198,5,93,158,237,46,118,181,144,83,11,200,187,120,32,227,106,169,241,50,65,130,218,25,60,255,167,100,23,212,140,79,0,196,149,81,55,243,162,102,110,170,251,63,89,157,204,8,220,24,73,141,235,47,126,186,178,118,39,227,133,65,16,212,165,97,48,244,146,86,7,195,203,15,94,154,252,56,105,173,121,189,236,40,78,138,219,31,23,211,130,70,32,228,181,113,87,147,194,6,96,164,245,49,57,253,172,104,14,202,155,95,139,79,30,218,188,120,41,237,229,33,112,180,210,22,71,131,242,54,103,163,197,1,80,148,156,88,9,205,171,111,62,250,46,234,187,127,25,221,140,72,64,132,213,17,119,179,226,38,174,106,59,255,153,93,12,200,192,4,85,145,247,51,98,166,114,182,231,35,69,129,208,20,28,216,137,77,43,239,190,122,11,207,158,90,60,248,169,109,101,161,240,52,82,150,199,3,215,19,66,134,224,36,117,177,185,125,44,232,142,74,27,223,249,61,108,168,206,10,91,159,151,83,2,198,160,100,53,241,37,225,176,116,18,214,135,67,75,143,222,26,124,184,233,45,92,152,201,13,107,175,254,58,50,246,167,99,5,193,144,84,128,68,21,209,183,115,34,230,238,42,123,191,217,29,76,136,0,197,151,82,51,246,164,97,102,163,241,52,85,144,194,7,204,9,91,158,255,58,104,173,170,111,61,248,153,92,14,203,133,64,18,215,182,115,33,228,227,38,116,177,208,21,71,130,73,140,222,27,122,191,237,40,47,234,184,125,28,217,139,78,23,210,128,69,36,225,179,118,113,180,230,35,66,135,213,16,219,30,76,137,232,45,127,186,189,120,42,239,142,75,25,220,146,87,5,192,161,100,54,243,244,49,99,166,199,2,80,149,94,155,201,12,109,168,250,63,56,253,175,106,11,206,156,89,46,235,185,124,29,216,138,79,72,141,223,26,123,190,236,41,226,39,117,176,209,20,70,131,132,65,19,214,183,114,32,229,171,110,60,249,152,93,15,202,205,8,90,159,254,59,105,172,103,162,240,53,84,145,195,6,1,196,150,83,50,247,165,96,57,252,174,107,10,207,157,88,95,154,200,13,108,169,251,62,245,48,98,167,198,3,81,148,147,86,4,193,160,101,55,242,188,121,43,238,143,74,24,221,218,31,77,136,233,44,126,187,112,181,231,34,67,134,212,17,22,211,129,68,37,224,178,119,0,198,145,87,63,249,174,104,126,184,239,41,65,135,208,22,252,58,109,171,195,5,82,148,130,68,19,213,189,123,44,234,229,35,116,178,218,28,75,141,155,93,10,204,164,98,53,243,25,223,136,78,38,224,183,113,103,161,246,48,88,158,201,15,215,17,70,128,232,46,121,191,169,111,56,254,150,80,7,193,43,237,186,124,20,210,133,67,85,147,196,2,106,172,251,61,50,244,163,101,13,203,156,90,76,138,221,27,115,181,226,36,206,8,95,153,241,55,96,166,176,118,33,231,143,73,30,216,179,117,34,228,140,74,29,219,205,11,92,154,242,52,99,165,79,137,222,24,112,182,225,39,49,247,160,102,14,200,159,89,86,144,199,1,105,175,248,62,40,238,185,127,23,209,134,64,170,108,59,253,149,83,4,194,212,18,69,131,235,45,122,188,100,162,245,51,91,157,202,12,26,220,139,77,37,227,180,114,152,94,9,207,167,97,54,240,230,32,119,177,217,31,72,142,129,71,16,214,190,120,47,233,255,57,110,168,192,6,81,151,125,187,236,42,66,132,211,21,3,197,146,84,60,250,173,107,0,199,147,84,59,252,168,111,118,177,229,34,77,138,222,25,236,43,127,184,215,16,68,131,154,93,9,206,161,102,50,245,197,2,86,145,254,57,109,170,179,116,32,231,136,79,27,220,41,238,186,125,18,213,129,70,95,152,204,11,100,163,247,48,151,80,4,195,172,107,63,248,225,38,114,181,218,29,73,142,123,188,232,47,64,135,211,20,13,202,158,89,54,241,165,98,82,149,193,6,105,174,250,61,36,227,183,112,31,216,140,75,190,121,45,234,133,66,22,209,200,15,91,156,243,52,96,167,51,244,160,103,8,207,155,92,69,130,214,17,126,185,237,42,223,24,76,139,228,35,119,176,169,110,58,253,146,85,1,198,246,49,101,162,205,10,94,153,128,71,19,212,187,124,40,239,26,221,137,78,33,230,178,117,108,171,255,56,87,144,196,3,164,99,55,240,159,88,12,203,210,21,65,134,233,46,122,189,72,143,219,28,115,180,224,39,62,249,173,106,5,194,150,81,97,166,242,53,90,157,201,14,23,208,132,67,44,235,191,120,141,74,30,217,182,113,37,226,251,60,104,175,192,7,83,148,0,200,141,69,7,207,138,66,14,198,131,75,9,193,132,76,28,212,145,89,27,211,150,94,18,218,159,87,21,221,152,80,56,240,181,125,63,247,178,122,54,254,187,115,49,249,188,116,36,236,169,97,35,235,174,102,42,226,167,111,45,229,160,104,112,184,253,53,119,191,250,50,126,182,243,59,121,177,244,60,108,164,225,41,107,163,230,46,98,170,239,39,101,173,232,32,72,128,197,13,79,135,194,10,70,142,203,3,65,137,204,4,84,156,217,17,83,155,222,22,90,146,215,31,93,149,208,24,224,40,109,165,231,47,106,162,238,38,99,171,233,33,100,172,252,52,113,185,251,51,118,190,242,58,127,183,245,61,120,176,216,16,85,157,223,23,82,154,214,30,91,147,209,25,92,148,196,12,73,129,195,11,78,134,202,2,71,143,205,5,64,136,144,88,29,213,151,95,26,210,158,86,19,219,153,81,20,220,140,68,1,201,139,67,6,206,130,74,15,199,133,77,8,192,168,96,37,237,175,103,34,234,166,110,43,227,161,105,44,228,180,124,57,241,179,123,62,246,186,114,55,255,189,117,48,248,0,201,143,70,3,202,140,69,6,207,137,64,5,204,138,67,12,197,131,74,15,198,128,73,10,195,133,76,9,192,134,79,24,209,151,94,27,210,148,93,30,215,145,88,29,212,146,91,20,221,155,82,23,222,152,81,18,219,157,84,17,216,158,87,48,249,191,118,51,250,188,117,54,255,185,112,53,252,186,115,60,245,179,122,63,246,176,121,58,243,181,124,57,240,182,127,40,225,167,110,43,226,164,109,46,231,161,104,45,228,162,107,36,237,171,98,39,238,168,97,34,235,173,100,33,232,174,103,96,169,239,38,99,170,236,37,102,175,233,32,101,172,234,35,108,165,227,42,111,166,224,41,106,163,229,44,105,160,230,47,120,177,247,62,123,178,244,61,126,183,241,56,125,180,242,59,116,189,251,50,119,190,248,49,114,187,253,52,113,184,254,55,80,153,223,22,83,154,220,21,86,159,217,16,85,156,218,19,92,149,211,26,95,150,208,25,90,147,213,28,89,144,214,31,72,129,199,14,75,130,196,13,78,135,193,8,77,132,194,11,68,141,203,2,71,142,200,1,66,139,205,4,65,136,206,7,0,202,137,67,15,197,134,76,30,212,151,93,17,219,152,82,60,246,181,127,51,249,186,112,34,232,171,97,45,231,164,110,120,178,241,59,119,189,254,52,102,172,239,37,105,163,224,42,68,142,205,7,75,129,194,8,90,144,211,25,85,159,220,22,240,58,121,179,255,53,118,188,238,36,103,173,225,43,104,162,204,6,69,143,195,9,74,128,210,24,91,145,221,23,84,158,136,66,1,203,135,77,14,196,150,92,31,213,153,83,16,218,180,126,61,247,187,113,50,248,170,96,35,233,165,111,44,230,253,55,116,190,242,56,123,177,227,41,106,160,236,38,101,175,193,11,72,130,206,4,71,141,223,21,86,156,208,26,89,147,133,79,12,198,138,64,3,201,155,81,18,216,148,94,29,215,185,115,48,250,182,124,63,245,167,109,46,228,168,98,33,235,13,199,132,78,2,200,139,65,19,217,154,80,28,214,149,95,49,251,184,114,62,244,183,125,47,229,166,108,32,234,169,99,117,191,252,54,122,176,243,57,107,161,226,40,100,174,237,39,73,131,192,10,70,140,207,5,87,157,222,20,88,146,209,27,0,203,139,64,11,192,128,75,22,221,157,86,29,214,150,93,44,231,167,108,39,236,172,103,58,241,177,122,49,250,186,113,88,147,211,24,83,152,216,19,78,133,197,14,69,142,206,5,116,191,255,52,127,180,244,63,98,169,233,34,105,162,226,41,176,123,59,240,187,112,48,251,166,109,45,230,173,102,38,237,156,87,23,220,151,92,28,215,138,65,1,202,129,74,10,193,232,35,99,168,227,40,104,163,254,53,117,190,245,62,126,181,196,15,79,132,207,4,68,143,210,25,89,146,217,18,82,153,125,182,246,61,118,189,253,54,107,160,224,43,96,171,235,32,81,154,218,17,90,145,209,26,71,140,204,7,76,135,199,12,37,238,174,101,46,229,165,110,51,248,184,115,56,243,179,120,9,194,130,73,2,201,137,66,31,212,148,95,20,223,159,84,205,6,70,141,198,13,77,134,219,16,80,155,208,27,91,144,225,42,106,161,234,33,97,170,247,60,124,183,252,55,119,188,149,94,30,213,158,85,21,222,131,72,8,195,136,67,3,200,185,114,50,249,178,121,57,242,175,100,36,239,164,111,47,228,0,204,133,73,23,219,146,94,46,226,171,103,57,245,188,112,92,144,217,21,75,135,206,2,114,190,247,59,101,169,224,44,184,116,61,241,175,99,42,230,150,90,19,223,129,77,4,200,228,40,97,173,243,63,118,186,202,6,79,131,221,17,88,148,109,161,232,36,122,182,255,51,67,143,198,10,84,152,209,29,49,253,180,120,38,234,163,111,31,211,154,86,8,196,141,65,213,25,80,156,194,14,71,139,251,55,126,178,236,32,105,165,137,69,12,192,158,82,27,215,167,107,34,238,176,124,53,249,218,22,95,147,205,1,72,132,244,56,113,189,227,47,102,170,134,74,3,207,145,93,20,216,168,100,45,225,191,115,58,246,98,174,231,43,117,185,240,60,76,128,201,5,91,151,222,18,62,242,187,119,41,229,172,96,16,220,149,89,7,203,130,78,183,123,50,254,160,108,37,233,153,85,28,208,142,66,11,199,235,39,110,162,252,48,121,181,197,9,64,140,210,30,87,155,15,195,138,70,24,212,157,81,33,237,164,104,54,250,179,127,83,159,214,26,68,136,193,13,125,177,248,52,106,166,239,35,0,205,135,74,19,222,148,89,38,235,161,108,53,248,178,127,76,129,203,6,95,146,216,21,106,167,237,32,121,180,254,51,152,85,31,210,139,70,12,193,190,115,57,244,173,96,42,231,212,25,83,158,199,10,64,141,242,63,117,184,225,44,102,171,45,224,170,103,62,243,185,116,11,198,140,65,24,213,159,82,97,172,230,43,114,191,245,56,71,138,192,13,84,153,211,30,181,120,50,255,166,107,33,236,147,94,20,217,128,77,7,202,249,52,126,179,234,39,109,160,223,18,88,149,204,1,75,134,90,151,221,16,73,132,206,3,124,177,251,54,111,162,232,37,22,219,145,92,5,200,130,79,48,253,183,122,35,238,164,105,194,15,69,136,209,28,86,155,228,41,99,174,247,58,112,189,142,67,9,196,157,80,26,215,168,101,47,226,187,118,60,241,119,186,240,61,100,169,227,46,81,156,214,27,66,143,197,8,59,246,188,113,40,229,175,98,29,208,154,87,14,195,137,68,239,34,104,165,252,49,123,182,201,4,78,131,218,23,93,144,163,110,36,233,176,125,55,250,133,72,2,207,150,91,17,220,0,206,129,79,31,209,158,80,62,240,191,113,33,239,160,110,124,178,253,51,99,173,226,44,66,140,195,13,93,147,220,18,248,54,121,183,231,41,102,168,198,8,71,137,217,23,88,150,132,74,5,203,155,85,26,212,186,116,59,245,165,107,36,234,237,35,108,162,242,60,115,189,211,29,82,156,204,2,77,131,145,95,16,222,142,64,15,193,175,97,46,224,176,126,49,255,21,219,148,90,10,196,139,69,43,229,170,100,52,250,181,123,105,167,232,38,118,184,247,57,87,153,214,24,72,134,201,7,199,9,70,136,216,22,89,151,249,55,120,182,230,40,103,169,187,117,58,244,164,106,37,235,133,75,4,202,154,84,27,213,63,241,190,112,32,238,161,111,1,207,128,78,30,208,159,81,67,141,194,12,92,146,221,19,125,179,252,50,98,172,227,45,42,228,171,101,53,251,180,122,20,218,149,91,11,197,138,68,86,152,215,25,73,135,200,6,104,166,233,39,119,185,246,56,210,28,83,157,205,3,76,130,236,34,109,163,243,61,114,188,174,96,47,225,177,127,48,254,144,94,17,223,143,65,14,192,0,207,131,76,27,212,152,87,54,249,181,122,45,226,174,97,108,163,239,32,119,184,244,59,90,149,217,22,65,142,194,13,216,23,91,148,195,12,64,143,238,33,109,162,245,58,118,185,180,123,55,248,175,96,44,227,130,77,1,206,153,86,26,213,173,98,46,225,182,121,53,250,155,84,24,215,128,79,3,204,193,14,66,141,218,21,89,150,247,56,116,187,236,35,111,160,117,186,246,57,110,161,237,34,67,140,192,15,88,151,219,20,25,214,154,85,2,205,129,78,47,224,172,99,52,251,183,120,71,136,196,11,92,147,223,16,113,190,242,61,106,165,233,38,43,228,168,103,48,255,179,124,29,210,158,81,6,201,133,74,159,80,28,211,132,75,7,200,169,102,42,229,178,125,49,254,243,60,112,191,232,39,107,164,197,10,70,137,222,17,93,146,234,37,105,166,241,62,114,189,220,19,95,144,199,8,68,139,134,73,5,202,157,82,30,209,176,127,51,252,171,100,40,231,50,253,177,126,41,230,170,101,4,203,135,72,31,208,156,83,94,145,221,18,69,138,198,9,104,167,235,36,115,188,240,63,0,208,189,109,103,183,218,10,206,30,115,163,169,121,20,196,129,81,60,236,230,54,91,139,79,159,242,34,40,248,149,69,31,207,162,114,120,168,197,21,209,1,108,188,182,102,11,219,158,78,35,243,249,41,68,148,80,128,237,61,55,231,138,90,62,238,131,83,89,137,228,52,240,32,77,157,151,71,42,250,191,111,2,210,216,8,101,181,113,161,204,28,22,198,171,123,33,241,156,76,70,150,251,43,239,63,82,130,136,88,53,229,160,112,29,205,199,23,122,170,110,190,211,3,9,217,180,100,124,172,193,17,27,203,166,118,178,98,15,223,213,5,104,184,253,45,64,144,154,74,39,247,51,227,142,94,84,132,233,57,99,179,222,14,4,212,185,105,173,125,16,192,202,26,119,167,226,50,95,143,133,85,56,232,44,252,145,65,75,155,246,38,66,146,255,47,37,245,152,72,140,92,49,225,235,59,86,134,195,19,126,174,164,116,25,201,13,221,176,96,106,186,215,7,93,141,224,48,58,234,135,87,147,67,46,254,244,36,73,153,220,12,97,177,187,107,6,214,18,194,175,127,117,165,200,24,0,209,191,110,99,178,220,13,198,23,121,168,165,116,26,203,145,64,46,255,242,35,77,156,87,134,232,57,52,229,139,90,63,238,128,81,92,141,227,50,249,40,70,151,154,75,37,244,174,127,17,192,205,28,114,163,104,185,215,6,11,218,180,101,126,175,193,16,29,204,162,115,184,105,7,214,219,10,100,181,239,62,80,129,140,93,51,226,41,248,150,71,74,155,245,36,65,144,254,47,34,243,157,76,135,86,56,233,228,53,91,138,208,1,111,190,179,98,12,221,22,199,169,120,117,164,202,27,252,45,67,146,159,78,32,241,58,235,133,84,89,136,230,55,109,188,210,3,14,223,177,96,171,122,20,197,200,25,119,166,195,18,124,173,160,113,31,206,5,212,186,107,102,183,217,8,82,131,237,60,49,224,142,95,148,69,43,250,247,38,72,153,130,83,61,236,225,48,94,143,68,149,251,42,39,246,152,73,19,194,172,125,112,161,207,30,213,4,106,187,182,103,9,216,189,108,2,211,222,15,97,176,123,170,196,21,24,201,167,118,44,253,147,66,79,158,240,33,234,59,85,132,137,88,54,231,0,210,185,107,111,189,214,4,222,12,103,181,177,99,8,218,161,115,24,202,206,28,119,165,127,173,198,20,16,194,169,123,95,141,230,52,48,226,137,91,129,83,56,234,238,60,87,133,254,44,71,149,145,67,40,250,32,242,153,75,79,157,246,36,190,108,7,213,209,3,104,186,96,178,217,11,15,221,182,100,31,205,166,116,112,162,201,27,193,19,120,170,174,124,23,197,225,51,88,138,142,92,55,229,63,237,134,84,80,130,233,59,64,146,249,43,47,253,150,68,158,76,39,245,241,35,72,154,97,179,216,10,14,220,183,101,191,109,6,212,208,2,105,187,192,18,121,171,175,125,22,196,30,204,167,117,113,163,200,26,62,236,135,85,81,131,232,58,224,50,89,139,143,93,54,228,159,77,38,244,240,34,73,155,65,147,248,42,46,252,151,69,223,13,102,180,176,98,9,219,1,211,184,106,110,188,215,5,126,172,199,21,17,195,168,122,160,114,25,203,207,29,118,164,128,82,57,235,239,61,86,132,94,140,231,53,49,227,136,90,33,243,152,74,78,156,247,37,255,45,70,148,144,66,41,251,0,211,187,104,107,184,208,3,214,5,109,190,189,110,6,213,177,98,10,217,218,9,97,178,103,180,220,15,12,223,183,100,127,172,196,23,20,199,175,124,169,122,18,193,194,17,121,170,206,29,117,166,165,118,30,205,24,203,163,112,115,160,200,27,254,45,69,150,149,70,46,253,40,251,147,64,67,144,248,43,79,156,244,39,36,247,159,76,153,74,34,241,242,33,73,154,129,82,58,233,234,57,81,130,87,132,236,63,60,239,135,84,48,227,139,88,91,136,224,51,230,53,93,142,141,94,54,229,225,50,90,137,138,89,49,226,55,228,140,95,92,143,231,52,80,131,235,56,59,232,128,83,134,85,61,238,237,62,86,133,158,77,37,246,245,38,78,157,72,155,243,32,35,240,152,75,47,252,148,71,68,151,255,44,249,42,66,145,146,65,41,250,31,204,164,119,116,167,207,28,201,26,114,161,162,113,25,202,174,125,21,198,197,22,126,173,120,171,195,16,19,192,168,123,96,179,219,8,11,216,176,99,182,101,13,222,221,14,102,181,209,2,106,185,186,105,1,210,7,212,188,111,108,191,215,4,0,212,181,97,119,163,194,22,238,58,91,143,153,77,44,248,193,21,116,160,182,98,3,215,47,251,154,78,88,140,237,57,159,75,42,254,232,60,93,137,113,165,196,16,6,210,179,103,94,138,235,63,41,253,156,72,176,100,5,209,199,19,114,166,35,247,150,66,84,128,225,53,205,25,120,172,186,110,15,219,226,54,87,131,149,65,32,244,12,216,185,109,123,175,206,26,188,104,9,221,203,31,126,170,82,134,231,51,37,241,144,68,125,169,200,28,10,222,191,107,147,71,38,242,228,48,81,133,70,146,243,39,49,229,132,80,168,124,29,201,223,11,106,190,135,83,50,230,240,36,69,145,105,189,220,8,30,202,171,127,217,13,108,184,174,122,27,207,55,227,130,86,64,148,245,33,24,204,173,121,111,187,218,14,246,34,67,151,129,85,52,224,101,177,208,4,18,198,167,115,139,95,62,234,252,40,73,157,164,112,17,197,211,7,102,178,74,158,255,43,61,233,136,92,250,46,79,155,141,89,56,236,20,192,161,117,99,183,214,2,59,239,142,90,76,152,249,45,213,1,96,180,162,118,23,195,0,213,183,98,115,166,196,17,230,51,81,132,149,64,34,247,209,4,102,179,162,119,21,192,55,226,128,85,68,145,243,38,191,106,8,221,204,25,123,174,89,140,238,59,42,255,157,72,110,187,217,12,29,200,170,127,136,93,63,234,251,46,76,153,99,182,212,1,16,197,167,114,133,80,50,231,246,35,65,148,178,103,5,208,193,20,118,163,84,129,227,54,39,242,144,69,220,9,107,190,175,122,24,205,58,239,141,88,73,156,254,43,13,216,186,111,126,171,201,28,235,62,92,137,152,77,47,250,198,19,113,164,181,96,2,215,32,245,151,66,83,134,228,49,23,194,160,117,100,177,211,6,241,36,70,147,130,87,53,224,121,172,206,27,10,223,189,104,159,74,40,253,236,57,91,142,168,125,31,202,219,14,108,185,78,155,249,44,61,232,138,95,165,112,18,199,214,3,97,180,67,150,244,33,48,229,135,82,116,161,195,22,7,210,176,101,146,71,37,240,225,52,86,131,26,207,173,120,105,188,222,11,252,41,75,158,143,90,56,237,203,30,124,169,184,109,15,218,45,248,154,79,94,139,233,60,0,214,177,103,127,169,206,24,254,40,79,153,129,87,48,230,225,55,80,134,158,72,47,249,31,201,174,120,96,182,209,7,223,9,110,184,160,118,17,199,33,247,144,70,94,136,239,57,62,232,143,89,65,151,240,38,192,22,113,167,191,105,14,216,163,117,18,196,220,10,109,187,93,139,236,58,34,244,147,69,66,148,243,37,61,235,140,90,188,106,13,219,195,21,114,164,124,170,205,27,3,213,178,100,130,84,51,229,253,43,76,154,157,75,44,250,226,52,83,133,99,181,210,4,28,202,173,123,91,141,234,60,36,242,149,67,165,115,20,194,218,12,107,189,186,108,11,221,197,19,116,162,68,146,245,35,59,237,138,92,132,82,53,227,251,45,74,156,122,172,203,29,5,211,180,98,101,179,212,2,26,204,171,125,155,77,42,252,228,50,85,131,248,46,73,159,135,81,54,224,6,208,183,97,121,175,200,30,25,207,168,126,102,176,215,1,231,49,86,128,152,78,41,255,39,241,150,64,88,142,233,63,217,15,104,190,166,112,23,193,198,16,119,161,185,111,8,222,56,238,137,95,71,145,246,32,0,215,179,100,123,172,200,31,246,33,69,146,141,90,62,233,241,38,66,149,138,93,57,238,7,208,180,99,124,171,207,24,255,40,76,155,132,83,55,224,9,222,186,109,114,165,193,22,14,217,189,106,117,162,198,17,248,47,75,156,131,84,48,231,227,52,80,135,152,79,43,252,21,194,166,113,110,185,221,10,18,197,161,118,105,190,218,13,228,51,87,128,159,72,44,251,28,203,175,120,103,176,212,3,234,61,89,142,145,70,34,245,237,58,94,137,150,65,37,242,27,204,168,127,96,183,211,4,219,12,104,191,160,119,19,196,45,250,158,73,86,129,229,50,42,253,153,78,81,134,226,53,220,11,111,184,167,112,20,195,36,243,151,64,95,136,236,59,210,5,97,182,169,126,26,205,213,2,102,177,174,121,29,202,35,244,144,71,88,143,235,60,56,239,139,92,67,148,240,39,206,25,125,170,181,98,6,209,201,30,122,173,178,101,1,214,63,232,140,91,68,147,247,32,199,16,116,163,188,107,15,216,49,230,130,85,74,157,249,46,54,225,133,82,77,154,254,41,192,23,115,164,187,108,8,223,0,216,173,117,71,159,234,50,142,86,35,251,201,17,100,188,1,217,172,116,70,158,235,51,143,87,34,250,200,16,101,189,2,218,175,119,69,157,232,48,140,84,33,249,203,19,102,190,3,219,174,118,68,156,233,49,141,85,32,248,202,18,103,191,4,220,169,113,67,155,238,54,138,82,39,255,205,21,96,184,5,221,168,112,66,154,239,55,139,83,38,254,204,20,97,185,6,222,171,115,65,153,236,52,136,80,37,253,207,23,98,186,7,223,170,114,64,152,237,53,137,81,36,252,206,22,99,187,8,208,165,125,79,151,226,58,134,94,43,243,193,25,108,180,9,209,164,124,78,150,227,59,135,95,42,242,192,24,109,181,10,210,167,127,77,149,224,56,132,92,41,241,195,27,110,182,11,211,166,126,76,148,225,57,133,93,40,240,194,26,111,183,12,212,161,121,75,147,230,62,130,90,47,247,197,29,104,176,13,213,160,120,74,146,231,63,131,91,46,246,196,28,105,177,14,214,163,123,73,145,228,60,128,88,45,245,199,31,106,178,15,215,162,122,72,144,229,61,129,89,44,244,198,30,107,179,0,217,175,118,67,154,236,53,134,95,41,240,197,28,106,179,17,200,190,103,82,139,253,36,151,78,56,225,212,13,123,162,34,251,141,84,97,184,206,23,164,125,11,210,231,62,72,145,51,234,156,69,112,169,223,6,181,108,26,195,246,47,89,128,68,157,235,50,7,222,168,113,194,27,109,180,129,88,46,247,85,140,250,35,22,207,185,96,211,10,124,165,144,73,63,230,102,191,201,16,37,252,138,83,224,57,79,150,163,122,12,213,119,174,216,1,52,237,155,66,241,40,94,135,178,107,29,196,136,81,39,254,203,18,100,189,14,215,161,120,77,148,226,59,153,64,54,239,218,3,117,172,31,198,176,105,92,133,243,42,170,115,5,220,233,48,70,159,44,245,131,90,111,182,192,25,187,98,20,205,248,33,87,142,61,228,146,75,126,167,209,8,204,21,99,186,143,86,32,249,74,147,229,60,9,208,166,127,221,4,114,171,158,71,49,232,91,130,244,45,24,193,183,110,238,55,65,152,173,116,2,219,104,177,199,30,43,242,132,93,255,38,80,137,188,101,19,202,121,160,214,15,58,227,149,76,0,218,169,115,79,149,230,60,158,68,55,237,209,11,120,162,33,251,136,82,110,180,199,29,191,101,22,204,240,42,89,131,66,152,235,49,13,215,164,126,220,6,117,175,147,73,58,224,99,185,202,16,44,246,133,95,253,39,84,142,178,104,27,193,132,94,45,247,203,17,98,184,26,192,179,105,85,143,252,38,165,127,12,214,234,48,67,153,59,225,146,72,116,174,221,7,198,28,111,181,137,83,32,250,88,130,241,43,23,205,190,100,231,61,78,148,168,114,1,219,121,163,208,10,54,236,159,69,21,207,188,102,90,128,243,41,139,81,34,248,196,30,109,183,52,238,157,71,123,161,210,8,170,112,3,217,229,63,76,150,87,141,254,36,24,194,177,107,201,19,96,186,134,92,47,245,118,172,223,5,57,227,144,74,232,50,65,155,167,125,14,212,145,75,56,226,222,4,119,173,15,213,166,124,64,154,233,51,176,106,25,195,255,37,86,140,46,244,135,93,97,187,200,18,211,9,122,160,156,70,53,239,77,151,228,62,2,216,171,113,242,40,91,129,189,103,20,206,108,182,197,31,35,249,138,80,0,219,171,112,75,144,224,59,150,77,61,230,221,6,118,173,49,234,154,65,122,161,209,10,167,124,12,215,236,55,71,156,98,185,201,18,41,242,130,89,244,47,95,132,191,100,20,207,83,136,248,35,24,195,179,104,197,30,110,181,142,85,37,254,196,31,111,180,143,84,36,255,82,137,249,34,25,194,178,105,245,46,94,133,190,101,21,206,99,184,200,19,40,243,131,88,166,125,13,214,237,54,70,157,48,235,155,64,123,160,208,11,151,76,60,231,220,7,119,172,1,218,170,113,74,145,225,58,149,78,62,229,222,5,117,174,3,216,168,115,72,147,227,56,164,127,15,212,239,52,68,159,50,233,153,66,121,162,210,9,247,44,92,135,188,103,23,204,97,186,202,17,42,241,129,90,198,29,109,182,141,86,38,253,80,139,251,32,27,192,176,107,81,138,250,33,26,193,177,106,199,28,108,183,140,87,39,252,96,187,203,16,43,240,128,91,246,45,93,134,189,102,22,205,51,232,152,67,120,163,211,8,165,126,14,213,238,53,69,158,2,217,169,114,73,146,226,57,148,79,63,228,223,4,116,175,0,220,165,121,87,139,242,46,174,114,11,215,249,37,92,128,65,157,228,56,22,202,179,111,239,51,74,150,184,100,29,193,130,94,39,251,213,9,112,172,44,240,137,85,123,167,222,2,195,31,102,186,148,72,49,237,109,177,200,20,58,230,159,67,25,197,188,96,78,146,235,55,183,107,18,206,224,60,69,153,88,132,253,33,15,211,170,118,246,42,83,143,161,125,4,216,155,71,62,226,204,16,105,181,53,233,144,76,98,190,199,27,218,6,127,163,141,81,40,244,116,168,209,13,35,255,134,90,50,238,151,75,101,185,192,28,156,64,57,229,203,23,110,178,115,175,214,10,36,248,129,93,221,1,120,164,138,86,47,243,176,108,21,201,231,59,66,158,30,194,187,103,73,149,236,48,241,45,84,136,166,122,3,223,95,131,250,38,8,212,173,113,43,247,142,82,124,160,217,5,133,89,32,252,210,14,119,171,106,182,207,19,61,225,152,68,196,24,97,189,147,79,54,234,169,117,12,208,254,34,91,135,7,219,162,126,80,140,245,41,232,52,77,145,191,99,26,198,70,154,227,63,17,205,180,104,0,221,167,122,83,142,244,41,166,123,1,220,245,40,82,143,81,140,246,43,2,223,165,120,247,42,80,141,164,121,3,222,162,127,5,216,241,44,86,139,4,217,163,126,87,138,240,45,243,46,84,137,160,125,7,218,85,136,242,47,6,219,161,124,89,132,254,35,10,215,173,112,255,34,88,133,172,113,11,214,8,213,175,114,91,134,252,33,174,115,9,212,253,32,90,135,251,38,92,129,168,117,15,210,93,128,250,39,14,211,169,116,170,119,13,208,249,36,94,131,12,209,171,118,95,130,248,37,178,111,21,200,225,60,70,155,20,201,179,110,71,154,224,61,227,62,68,153,176,109,23,202,69,152,226,63,22,203,177,108,16,205,183,106,67,158,228,57,182,107,17,204,229,56,66,159,65,156,230,59,18,207,181,104,231,58,64,157,180,105,19,206,235,54,76,145,184,101,31,194,77,144,234,55,30,195,185,100,186,103,29,192,233,52,78,147,28,193,187,102,79,146,232,53,73,148,238,51,26,199,189,96,239,50,72,149,188,97,27,198,24,197,191,98,75,150,236,49,190,99,25,196,237,48,74,151,0,222,161,127,95,129,254,32,190,96,31,193,225,63,64,158,97,191,192,30,62,224,159,65,223,1,126,160,128,94,33,255,194,28,99,189,157,67,60,226,124,162,221,3,35,253,130,92,163,125,2,220,252,34,93,131,29,195,188,98,66,156,227,61,153,71,56,230,198,24,103,185,39,249,134,88,120,166,217,7,248,38,89,135,167,121,6,216,70,152,231,57,25,199,184,102,91,133,250,36,4,218,165,123,229,59,68,154,186,100,27,197,58,228,155,69,101,187,196,26,132,90,37,251,219,5,122,164,47,241,142,80,112,174,209,15,145,79,48,238,206,16,111,177,78,144,239,49,17,207,176,110,240,46,81,143,175,113,14,208,237,51,76,146,178,108,19,205,83,141,242,44,12,210,173,115,140,82,45,243,211,13,114,172,50,236,147,77,109,179,204,18,182,104,23,201,233,55,72,150,8,214,169,119,87,137,246,40,215,9,118,168,136,86,41,247,105,183,200,22,54,232,151,73,116,170,213,11,43,245,138,84,202,20,107,181,149,75,52,234,21,203,180,106,74,148,235,53,171,117,10,212,244,42,85,139,0,223,163,124,91,132,248,39,182,105,21,202,237,50,78,145,113,174,210,13,42,245,137,86,199,24,100,187,156,67,63,224,226,61,65,158,185,102,26,197,84,139,247,40,15,208,172,115,147,76,48,239,200,23,107,180,37,250,134,89,126,161,221,2,217,6,122,165,130,93,33,254,111,176,204,19,52,235,151,72,168,119,11,212,243,44,80,143,30,193,189,98,69,154,230,57,59,228,152,71,96,191,195,28,141,82,46,241,214,9,117,170,74,149,233,54,17,206,178,109,252,35,95,128,167,120,4,219,175,112,12,211,244,43,87,136,25,198,186,101,66,157,225,62,222,1,125,162,133,90,38,249,104,183,203,20,51,236,144,79,77,146,238,49,22,201,181,106,251,36,88,135,160,127,3,220,60,227,159,64,103,184,196,27,138,85,41,246,209,14,114,173,118,169,213,10,45,242,142,81,192,31,99,188,155,68,56,231,7,216,164,123,92,131,255,32,177,110,18,205,234,53,73,150,148,75,55,232,207,16,108,179,34,253,129,94,121,166,218,5,229,58,70,153,190,97,29,194,83,140,240,47,8,215,171,116,0,224,221,61,167,71,122,154,83,179,142,110,244,20,41,201,166,70,123,155,1,225,220,60,245,21,40,200,82,178,143,111,81,177,140,108,246,22,43,203,2,226,223,63,165,69,120,152,247,23,42,202,80,176,141,109,164,68,121,153,3,227,222,62,162,66,127,159,5,229,216,56,241,17,44,204,86,182,139,107,4,228,217,57,163,67,126,158,87,183,138,106,240,16,45,205,243,19,46,206,84,180,137,105,160,64,125,157,7,231,218,58,85,181,136,104,242,18,47,207,6,230,219,59,161,65,124,156,89,185,132,100,254,30,35,195,10,234,215,55,173,77,112,144,255,31,34,194,88,184,133,101,172,76,113,145,11,235,214,54,8,232,213,53,175,79,114,146,91,187,134,102,252,28,33,193,174,78,115,147,9,233,212,52,253,29,32,192,90,186,135,103,251,27,38,198,92,188,129,97,168,72,117,149,15,239,210,50,93,189,128,96,250,26,39,199,14,238,211,51,169,73,116,148,170,74,119,151,13,237,208,48,249,25,36,196,94,190,131,99,12,236,209,49,171,75,118,150,95,191,130,98,248,24,37,197,0,225,223,62,163,66,124,157,91,186,132,101,248,25,39,198,182,87,105,136,21,244,202,43,237,12,50,211,78,175,145,112,113,144,174,79,210,51,13,236,42,203,245,20,137,104,86,183,199,38,24,249,100,133,187,90,156,125,67,162,63,222,224,1,226,3,61,220,65,160,158,127,185,88,102,135,26,251,197,36,84,181,139,106,247,22,40,201,15,238,208,49,172,77,115,146,147,114,76,173,48,209,239,14,200,41,23,246,107,138,180,85,37,196,250,27,134,103,89,184,126,159,161,64,221,60,2,227,217,56,6,231,122,155,165,68,130,99,93,188,33,192,254,31,111,142,176,81,204,45,19,242,52,213,235,10,151,118,72,169,168,73,119,150,11,234,212,53,243,18,44,205,80,177,143,110,30,255,193,32,189,92,98,131,69,164,154,123,230,7,57,216,59,218,228,5,152,121,71,166,96,129,191,94,195,34,28,253,141,108,82,179,46,207,241,16,214,55,9,232,117,148,170,75,74,171,149,116,233,8,54,215,17,240,206,47,178,83,109,140,252,29,35,194,95,190,128,97,167,70,120,153,4,229,219,58,0,226,217,59,175,77,118,148,67,161,154,120,236,14,53,215,134,100,95,189,41,203,240,18,197,39,28,254,106,136,179,81,17,243,200,42,190,92,103,133,82,176,139,105,253,31,36,198,151,117,78,172,56,218,225,3,212,54,13,239,123,153,162,64,34,192,251,25,141,111,84,182,97,131,184,90,206,44,23,245,164,70,125,159,11,233,210,48,231,5,62,220,72,170,145,115,51,209,234,8,156,126,69,167,112,146,169,75,223,61,6,228,181,87,108,142,26,248,195,33,246,20,47,205,89,187,128,98,68,166,157,127,235,9,50,208,7,229,222,60,168,74,113,147,194,32,27,249,109,143,180,86,129,99,88,186,46,204,247,21,85,183,140,110,250,24,35,193,22,244,207,45,185,91,96,130,211,49,10,232,124,158,165,71,144,114,73,171,63,221,230,4,102,132,191,93,201,43,16,242,37,199,252,30,138,104,83,177,224,2,57,219,79,173,150,116,163,65,122,152,12,238,213,55,119,149,174,76,216,58,1,227,52,214,237,15,155,121,66,160,241,19,40,202,94,188,135,101,178,80,107,137,29,255,196,38,0,227,219,56,171,72,112,147,75,168,144,115,224,3,59,216,150,117,77,174,61,222,230,5,221,62,6,229,118,149,173,78,49,210,234,9,154,121,65,162,122,153,161,66,209,50,10,233,167,68,124,159,12,239,215,52,236,15,55,212,71,164,156,127,98,129,185,90,201,42,18,241,41,202,242,17,130,97,89,186,244,23,47,204,95,188,132,103,191,92,100,135,20,247,207,44,83,176,136,107,248,27,35,192,24,251,195,32,179,80,104,139,197,38,30,253,110,141,181,86,142,109,85,182,37,198,254,29,196,39,31,252,111,140,180,87,143,108,84,183,36,199,255,28,82,177,137,106,249,26,34,193,25,250,194,33,178,81,105,138,245,22,46,205,94,189,133,102,190,93,101,134,21,246,206,45,99,128,184,91,200,43,19,240,40,203,243,16,131,96,88,187,166,69,125,158,13,238,214,53,237,14,54,213,70,165,157,126,48,211,235,8,155,120,64,163,123,152,160,67,208,51,11,232,151,116,76,175,60,223,231,4,220,63,7,228,119,148,172,79,1,226,218,57,170,73,113,146,74,169,145,114,225,2,58,217,0,228,213,49,183,83,98,134,115,151,166,66,196,32,17,245,230,2,51,215,81,181,132,96,149,113,64,164,34,198,247,19,209,53,4,224,102,130,179,87,162,70,119,147,21,241,192,36,55,211,226,6,128,100,85,177,68,160,145,117,243,23,38,194,191,91,106,142,8,236,221,57,204,40,25,253,123,159,174,74,89,189,140,104,238,10,59,223,42,206,255,27,157,121,72,172,110,138,187,95,217,61,12,232,29,249,200,44,170,78,127,155,136,108,93,185,63,219,234,14,251,31,46,202,76,168,153,125,99,135,182,82,212,48,1,229,16,244,197,33,167,67,114,150,133,97,80,180,50,214,231,3,246,18,35,199,65,165,148,112,178,86,103,131,5,225,208,52,193,37,20,240,118,146,163,71,84,176,129,101,227,7,54,210,39,195,242,22,144,116,69,161,220,56,9,237,107,143,190,90,175,75,122,158,24,252,205,41,58,222,239,11,141,105,88,188,73,173,156,120,254,26,43,207,13,233,216,60,186,94,111,139,126,154,171,79,201,45,28,248,235,15,62,218,92,184,137,109,152,124,77,169,47,203,250,30,0,229,215,50,179,86,100,129,123,158,172,73,200,45,31,250,246,19,33,196,69,160,146,119,141,104,90,191,62,219,233,12,241,20,38,195,66,167,149,112,138,111,93,184,57,220,238,11,7,226,208,53,180,81,99,134,124,153,171,78,207,42,24,253,255,26,40,205,76,169,155,126,132,97,83,182,55,210,224,5,9,236,222,59,186,95,109,136,114,151,165,64,193,36,22,243,14,235,217,60,189,88,106,143,117,144,162,71,198,35,17,244,248,29,47,202,75,174,156,121,131,102,84,177,48,213,231,2,227,6,52,209,80,181,135,98,152,125,79,170,43,206,252,25,21,240,194,39,166,67,113,148,110,139,185,92,221,56,10,239,18,247,197,32,161,68,118,147,105,140,190,91,218,63,13,232,228,1,51,214,87,178,128,101,159,122,72,173,44,201,251,30,28,249,203,46,175,74,120,157,103,130,176,85,212,49,3,230,234,15,61,216,89,188,142,107,145,116,70,163,34,199,245,16,237,8,58,223,94,187,137,108,150,115,65,164,37,192,242,23,27,254,204,41,168,77,127,154,96,133,183,82,211,54,4,225,0,230,209,55,191,89,110,136,99,133,178,84,220,58,13,235,198,32,23,241,121,159,168,78,165,67,116,146,26,252,203,45,145,119,64,166,46,200,255,25,242,20,35,197,77,171,156,122,87,177,134,96,232,14,57,223,52,210,229,3,139,109,90,188,63,217,238,8,128,102,81,183,92,186,141,107,227,5,50,212,249,31,40,206,70,160,151,113,154,124,75,173,37,195,244,18,174,72,127,153,17,247,192,38,205,43,28,250,114,148,163,69,104,142,185,95,215,49,6,224,11,237,218,60,180,82,101,131,126,152,175,73,193,39,16,246,29,251,204,42,162,68,115,149,184,94,105,143,7,225,214,48,219,61,10,236,100,130,181,83,239,9,62,216,80,182,129,103,140,106,93,187,51,213,226,4,41,207,248,30,150,112,71,161,74,172,155,125,245,19,36,194,65,167,144,118,254,24,47,201,34,196,243,21,157,123,76,170,135,97,86,176,56,222,233,15,228,2,53,211,91,189,138,108,208,54,1,231,111,137,190,88,179,85,98,132,12,234,221,59,22,240,199,33,169,79,120,158,117,147,164,66,202,44,27,253,0,231,211,52,187,92,104,143,107,140,184,95,208,55,3,228,214,49,5,226,109,138,190,89,189,90,110,137,6,225,213,50,177,86,98,133,10,237,217,62,218,61,9,238,97,134,178,85,103,128,180,83,220,59,15,232,12,235,223,56,183,80,100,131,127,152,172,75,196,35,23,240,20,243,199,32,175,72,124,155,169,78,122,157,18,245,193,38,194,37,17,246,121,158,170,77,206,41,29,250,117,146,166,65,165,66,118,145,30,249,205,42,24,255,203,44,163,68,112,151,115,148,160,71,200,47,27,252,254,25,45,202,69,162,150,113,149,114,70,161,46,201,253,26,40,207,251,28,147,116,64,167,67,164,144,119,248,31,43,204,79,168,156,123,244,19,39,192,36,195,247,16,159,120,76,171,153,126,74,173,34,197,241,22,242,21,33,198,73,174,154,125,129,102,82,181,58,221,233,14,234,13,57,222,81,182,130,101,87,176,132,99,236,11,63,216,60,219,239,8,135,96,84,179,48,215,227,4,139,108,88,191,91,188,136,111,224,7,51,212,230,1,53,210,93,186,142,105,141,106,94,185,54,209,229,2,0,232,205,37,135,111,74,162,19,251,222,54,148,124,89,177,38,206,235,3,161,73,108,132,53,221,248,16,178,90,127,151,76,164,129,105,203,35,6,238,95,183,146,122,216,48,21,253,106,130,167,79,237,5,32,200,121,145,180,92,254,22,51,219,152,112,85,189,31,247,210,58,139,99,70,174,12,228,193,41,190,86,115,155,57,209,244,28,173,69,96,136,42,194,231,15,212,60,25,241,83,187,158,118,199,47,10,226,64,168,141,101,242,26,63,215,117,157,184,80,225,9,44,196,102,142,171,67,45,197,224,8,170,66,103,143,62,214,243,27,185,81,116,156,11,227,198,46,140,100,65,169,24,240,213,61,159,119,82,186,97,137,172,68,230,14,43,195,114,154,191,87,245,29,56,208,71,175,138,98,192,40,13,229,84,188,153,113,211,59,30,246,181,93,120,144,50,218,255,23,166,78,107,131,33,201,236,4,147,123,94,182,20,252,217,49,128,104,77,165,7,239,202,34,249,17,52,220,126,150,179,91,234,2,39,207,109,133,160,72,223,55,18,250,88,176,149,125,204,36,1,233,75,163,134,110,0,233,207,38,131,106,76,165,27,242,212,61,152,113,87,190,54,223,249,16,181,92,122,147,45,196,226,11,174,71,97,136,108,133,163,74,239,6,32,201,119,158,184,81,244,29,59,210,90,179,149,124,217,48,22,255,65,168,142,103,194,43,13,228,216,49,23,254,91,178,148,125,195,42,12,229,64,169,143,102,238,7,33,200,109,132,162,75,245,28,58,211,118,159,185,80,180,93,123,146,55,222,248,17,175,70,96,137,44,197,227,10,130,107,77,164,1,232,206,39,153,112,86,191,26,243,213,60,173,68,98,139,46,199,225,8,182,95,121,144,53,220,250,19,155,114,84,189,24,241,215,62,128,105,79,166,3,234,204,37,193,40,14,231,66,171,141,100,218,51,21,252,89,176,150,127,247,30,56,209,116,157,187,82,236,5,35,202,111,134,160,73,117,156,186,83,246,31,57,208,110,135,161,72,237,4,34,203,67,170,140,101,192,41,15,230,88,177,151,126,219,50,20,253,25,240,214,63,154,115,85,188,2,235,205,36,129,104,78,167,47,198,224,9,172,69,99,138,52,221,251,18,183,94,120,145,0,234,201,35,143,101,70,172,3,233,202,32,140,102,69,175,6,236,207,37,137,99,64,170,5,239,204,38,138,96,67,169,12,230,197,47,131,105,74,160,15,229,198,44,128,106,73,163,10,224,195,41,133,111,76,166,9,227,192,42,134,108,79,165,24,242,209,59,151,125,94,180,27,241,210,56,148,126,93,183,30,244,215,61,145,123,88,178,29,247,212,62,146,120,91,177,20,254,221,55,155,113,82,184,23,253,222,52,152,114,81,187,18,248,219,49,157,119,84,190,17,251,216,50,158,116,87,189,48,218,249,19,191,85,118,156,51,217,250,16,188,86,117,159,54,220,255,21,185,83,112,154,53,223,252,22,186,80,115,153,60,214,245,31,179,89,122,144,63,213,246,28,176,90,121,147,58,208,243,25,181,95,124,150,57,211,240,26,182,92,127,149,40,194,225,11,167,77,110,132,43,193,226,8,164,78,109,135,46,196,231,13,161,75,104,130,45,199,228,14,162,72,107,129,36,206,237,7,171,65,98,136,39,205,238,4,168,66,97,139,34,200,235,1,173,71,100,142,33,203,232,2,174,68,103,141,0,235,203,32,139,96,64,171,11,224,192,43,128,107,75,160,22,253,221,54,157,118,86,189,29,246,214,61,150,125,93,182,44,199,231,12,167,76,108,135,39,204,236,7,172,71,103,140,58,209,241,26,177,90,122,145,49,218,250,17,186,81,113,154,88,179,147,120,211,56,24,243,83,184,152,115,216,51,19,248,78,165,133,110,197,46,14,229,69,174,142,101,206,37,5,238,116,159,191,84,255,20,52,223,127,148,180,95,244,31,63,212,98,137,169,66,233,2,34,201,105,130,162,73,226,9,41,194,176,91,123,144,59,208,240,27,187,80,112,155,48,219,251,16,166,77,109,134,45,198,230,13,173,70,102,141,38,205,237,6,156,119,87,188,23,252,220,55,151,124,92,183,28,247,215,60,138,97,65,170,1,234,202,33,129,106,74,161,10,225,193,42,232,3,35,200,99,136,168,67,227,8,40,195,104,131,163,72,254,21,53,222,117,158,190,85,245,30,62,213,126,149,181,94,196,47,15,228,79,164,132,111,207,36,4,239,68,175,143,100,210,57,25,242,89,178,146,121,217,50,18,249,82,185,153,114,0,236,197,41,151,123,82,190,51,223,246,26,164,72,97,141,102,138,163,79,241,29,52,216,85,185,144,124,194,46,7,235,204,32,9,229,91,183,158,114,255,19,58,214,104,132,173,65,170,70,111,131,61,209,248,20,153,117,92,176,14,226,203,39,133,105,64,172,18,254,215,59,182,90,115,159,33,205,228,8,227,15,38,202,116,152,177,93,208,60,21,249,71,171,130,110,73,165,140,96,222,50,27,247,122,150,191,83,237,1,40,196,47,195,234,6,184,84,125,145,28,240,217,53,139,103,78,162,23,251,210,62,128,108,69,169,36,200,225,13,179,95,118,154,113,157,180,88,230,10,35,207,66,174,135,107,213,57,16,252,219,55,30,242,76,160,137,101,232,4,45,193,127,147,186,86,189,81,120,148,42,198,239,3,142,98,75,167,25,245,220,48,146,126,87,187,5,233,192,44,161,77,100,136,54,218,243,31,244,24,49,221,99,143,166,74,199,43,2,238,80,188,149,121,94,178,155,119,201,37,12,224,109,129,168,68,250,22,63,211,56,212,253,17,175,67,106,134,11,231,206,34,156,112,89,181,0,237,199,42,147,126,84,185,59,214,252,17,168,69,111,130,118,155,177,92,229,8,34,207,77,160,138,103,222,51,25,244,236,1,43,198,127,146,184,85,215,58,16,253,68,169,131,110,154,119,93,176,9,228,206,35,161,76,102,139,50,223,245,24,197,40,2,239,86,187,145,124,254,19,57,212,109,128,170,71,179,94,116,153,32,205,231,10,136,101,79,162,27,246,220,49,41,196,238,3,186,87,125,144,18,255,213,56,129,108,70,171,95,178,152,117,204,33,11,230,100,137,163,78,247,26,48,221,151,122,80,189,4,233,195,46,172,65,107,134,63,210,248,21,225,12,38,203,114,159,181,88,218,55,29,240,73,164,142,99,123,150,188,81,232,5,47,194,64,173,135,106,211,62,20,249,13,224,202,39,158,115,89,180,54,219,241,28,165,72,98,143,82,191,149,120,193,44,6,235,105,132,174,67,250,23,61,208,36,201,227,14,183,90,112,157,31,242,216,53,140,97,75,166,190,83,121,148,45,192,234,7,133,104,66,175,22,251,209,60,200,37,15,226,91,182,156,113,243,30,52,217,96,141,167,74,0,238,193,47,159,113,94,176,35,205,226,12,188,82,125,147,70,168,135,105,217,55,24,246,101,139,164,74,250,20,59,213,140,98,77,163,19,253,210,60,175,65,110,128,48,222,241,31,202,36,11,229,85,187,148,122,233,7,40,198,118,152,183,89,5,235,196,42,154,116,91,181,38,200,231,9,185,87,120,150,67,173,130,108,220,50,29,243,96,142,161,79,255,17,62,208,137,103,72,166,22,248,215,57,170,68,107,133,53,219,244,26,207,33,14,224,80,190,145,127,236,2,45,195,115,157,178,92,10,228,203,37,149,123,84,186,41,199,232,6,182,88,119,153,76,162,141,99,211,61,18,252,111,129,174,64,240,30,49,223,134,104,71,169,25,247,216,54,165,75,100,138,58,212,251,21,192,46,1,239,95,177,158,112,227,13,34,204,124,146,189,83,15,225,206,32,144,126,81,191,44,194,237,3,179,93,114,156,73,167,136,102,214,56,23,249,106,132,171,69,245,27,52,218,131,109,66,172,28,242,221,51,160,78,97,143,63,209,254,16,197,43,4,234,90,180,155,117,230,8,39,201,121,151,184,86,0,239,195,44,155,116,88,183,43,196,232,7,176,95,115,156,86,185,149,122,205,34,14,225,125,146,190,81,230,9,37,202,172,67,111,128,55,216,244,27,135,104,68,171,28,243,223,48,250,21,57,214,97,142,162,77,209,62,18,253,74,165,137,102,69,170,134,105,222,49,29,242,110,129,173,66,245,26,54,217,19,252,208,63,136,103,75,164,56,215,251,20,163,76,96,143,233,6,42,197,114,157,177,94,194,45,1,238,89,182,154,117,191,80,124,147,36,203,231,8,148,123,87,184,15,224,204,35,138,101,73,166,17,254,210,61,161,78,98,141,58,213,249,22,220,51,31,240,71,168,132,107,247,24,52,219,108,131,175,64,38,201,229,10,189,82,126,145,13,226,206,33,150,121,85,186,112,159,179,92,235,4,40,199,91,180,152,119,192,47,3,236,207,32,12,227,84,187,151,120,228,11,39,200,127,144,188,83,153,118,90,181,2,237,193,46,178,93,113,158,41,198,234,5,99,140,160,79,248,23,59,212,72,167,139,100,211,60,16,255,53,218,246,25,174,65,109,130,30,241,221,50,133,106,70,169,0,240,253,13,231,23,26,234,211,35,46,222,52,196,201,57,187,75,70,182,92,172,161,81,104,152,149,101,143,127,114,130,107,155,150,102,140,124,113,129,184,72,69,181,95,175,162,82,208,32,45,221,55,199,202,58,3,243,254,14,228,20,25,233,214,38,43,219,49,193,204,60,5,245,248,8,226,18,31,239,109,157,144,96,138,122,119,135,190,78,67,179,89,169,164,84,189,77,64,176,90,170,167,87,110,158,147,99,137,121,116,132,6,246,251,11,225,17,28,236,213,37,40,216,50,194,207,63,177,65,76,188,86,166,171,91,98,146,159,111,133,117,120,136,10,250,247,7,237,29,16,224,217,41,36,212,62,206,195,51,218,42,39,215,61,205,192,48,9,249,244,4,238,30,19,227,97,145,156,108,134,118,123,139,178,66,79,191,85,165,168,88,103,151,154,106,128,112,125,141,180,68,73,185,83,163,174,94,220,44,33,209,59,203,198,54,15,255,242,2,232,24,21,229,12,252,241,1,235,27,22,230,223,47,34,210,56,200,197,53,183,71,74,186,80,160,173,93,100,148,153,105,131,115,126,142,0,241,255,14,227,18,28,237,219,42,36,213,56,201,199,54,171,90,84,165,72,185,183,70,112,129,143,126,147,98,108,157,75,186,180,69,168,89,87,166,144,97,111,158,115,130,140,125,224,17,31,238,3,242,252,13,59,202,196,53,216,41,39,214,150,103,105,152,117,132,138,123,77,188,178,67,174,95,81,160,61,204,194,51,222,47,33,208,230,23,25,232,5,244,250,11,221,44,34,211,62,207,193,48,6,247,249,8,229,20,26,235,118,135,137,120,149,100,106,155,173,92,82,163,78,191,177,64,49,192,206,63,210,35,45,220,234,27,21,228,9,248,246,7,154,107,101,148,121,136,134,119,65,176,190,79,162,83,93,172,122,139,133,116,153,104,102,151,161,80,94,175,66,179,189,76,209,32,46,223,50,195,205,60,10,251,245,4,233,24,22,231,167,86,88,169,68,181,187,74,124,141,131,114,159,110,96,145,12,253,243,2,239,30,16,225,215,38,40,217,52,197,203,58,236,29,19,226,15,254,240,1,55,198,200,57,212,37,43,218,71,182,184,73,164,85,91,170,156,109,99,146,127,142,128,113,0,242,249,11,239,29,22,228,195,49,58,200,44,222,213,39,155,105,98,144,116,134,141,127,88,170,161,83,183,69,78,188,43,217,210,32,196,54,61,207,232,26,17,227,7,245,254,12,176,66,73,187,95,173,166,84,115,129,138,120,156,110,101,151,86,164,175,93,185,75,64,178,149,103,108,158,122,136,131,113,205,63,52,198,34,208,219,41,14,252,247,5,225,19,24,234,125,143,132,118,146,96,107,153,190,76,71,181,81,163,168,90,230,20,31,237,9,251,240,2,37,215,220,46,202,56,51,193,172,94,85,167,67,177,186,72,111,157,150,100,128,114,121,139,55,197,206,60,216,42,33,211,244,6,13,255,27,233,226,16,135,117,126,140,104,154,145,99,68,182,189,79,171,89,82,160,28,238,229,23,243,1,10,248,223,45,38,212,48,194,201,59,250,8,3,241,21,231,236,30,57,203,192,50,214,36,47,221,97,147,152,106,142,124,119,133,162,80,91,169,77,191,180,70,209,35,40,218,62,204,199,53,18,224,235,25,253,15,4,246,74,184,179,65,165,87,92,174,137,123,112,130,102,148,159,109,0,243,251,8,235,24,16,227,203,56,48,195,32,211,219,40,139,120,112,131,96,147,155,104,64,179,187,72,171,88,80,163,11,248,240,3,224,19,27,232,192,51,59,200,43,216,208,35,128,115,123,136,107,152,144,99,75,184,176,67,160,83,91,168,22,229,237,30,253,14,6,245,221,46,38,213,54,197,205,62,157,110,102,149,118,133,141,126,86,165,173,94,189,78,70,181,29,238,230,21,246,5,13,254,214,37,45,222,61,206,198,53,150,101,109,158,125,142,134,117,93,174,166,85,182,69,77,190,44,223,215,36,199,52,60,207,231,20,28,239,12,255,247,4,167,84,92,175,76,191,183,68,108,159,151,100,135,116,124,143,39,212,220,47,204,63,55,196,236,31,23,228,7,244,252,15,172,95,87,164,71,180,188,79,103,148,156,111,140,127,119,132,58,201,193,50,209,34,42,217,241,2,10,249,26,233,225,18,177,66,74,185,90,169,161,82,122,137,129,114,145,98,106,153,49,194,202,57,218,41,33,210,250,9,1,242,17,226,234,25,186,73,65,178,81,162,170,89,113,130,138,121,154,105,97,146,0,244,245,1,247,3,2,246,243,7,6,242,4,240,241,5,251,15,14,250,12,248,249,13,8,252,253,9,255,11,10,254,235,31,30,234,28,232,233,29,24,236,237,25,239,27,26,238,16,228,229,17,231,19,18,230,227,23,22,226,20,224,225,21,203,63,62,202,60,200,201,61,56,204,205,57,207,59,58,206,48,196,197,49,199,51,50,198,195,55,54,194,52,192,193,53,32,212,213,33,215,35,34,214,211,39,38,210,36,208,209,37,219,47,46,218,44,216,217,45,40,220,221,41,223,43,42,222,139,127,126,138,124,136,137,125,120,140,141,121,143,123,122,142,112,132,133,113,135,115,114,134,131,119,118,130,116,128,129,117,96,148,149,97,151,99,98,150,147,103,102,146,100,144,145,101,155,111,110,154,108,152,153,109,104,156,157,105,159,107,106,158,64,180,181,65,183,67,66,182,179,71,70,178,68,176,177,69,187,79,78,186,76,184,185,77,72,188,189,73,191,75,74,190,171,95,94,170,92,168,169,93,88,172,173,89,175,91,90,174,80,164,165,81,167,83,82,166,163,87,86,162,84,160,161,85,0,245,247,2,243,6,4,241,251,14,12,249,8,253,255,10,235,30,28,233,24,237,239,26,16,229,231,18,227,22,20,225,203,62,60,201,56,205,207,58,48,197,199,50,195,54,52,193,32,213,215,34,211,38,36,209,219,46,44,217,40,221,223,42,139,126,124,137,120,141,143,122,112,133,135,114,131,118,116,129,96,149,151,98,147,102,100,145,155,110,108,153,104,157,159,106,64,181,183,66,179,70,68,177,187,78,76,185,72,189,191,74,171,94,92,169,88,173,175,90,80,165,167,82,163,86,84,161,11,254,252,9,248,13,15,250,240,5,7,242,3,246,244,1,224,21,23,226,19,230,228,17,27,238,236,25,232,29,31,234,192,53,55,194,51,198,196,49,59,206,204,57,200,61,63,202,43,222,220,41,216,45,47,218,208,37,39,210,35,214,212,33,128,117,119,130,115,134,132,113,123,142,140,121,136,125,127,138,107,158,156,105,152,109,111,154,144,101,103,146,99,150,148,97,75,190,188,73,184,77,79,186,176,69,71,178,67,182,180,65,160,85,87,162,83,166,164,81,91,174,172,89,168,93,95,170,0,246,241,7,255,9,14,248,227,21,18,228,28,234,237,27,219,45,42,220,36,210,213,35,56,206,201,63,199,49,54,192,171,93,90,172,84,162,165,83,72,190,185,79,183,65,70,176,112,134,129,119,143,121,126,136,147,101,98,148,108,154,157,107,75,189,186,76,180,66,69,179,168,94,89,175,87,161,166,80,144,102,97,151,111,153,158,104,115,133,130,116,140,122,125,139,224,22,17,231,31,233,238,24,3,245,242,4,252,10,13,251,59,205,202,60,196,50,53,195,216,46,41,223,39,209,214,32,150,96,103,145,105,159,152,110,117,131,132,114,138,124,123,141,77,187,188,74,178,68,67,181,174,88,95,169,81,167,160,86,61,203,204,58,194,52,51,197,222,40,47,217,33,215,208,38,230,16,23,225,25,239,232,30,5,243,244,2,250,12,11,253,221,43,44,218,34,212,211,37,62,200,207,57,193,55,48,198,6,240,247,1,249,15,8,254,229,19,20,226,26,236,235,29,118,128,135,113,137,127,120,142,149,99,100,146,106,156,155,109,173,91,92,170,82,164,163,85,78,184,191,73,177,71,64,182,0,247,243,4,251,12,8,255,235,28,24,239,16,231,227,20,203,60,56,207,48,199,195,52,32,215,211,36,219,44,40,223,139,124,120,143,112,135,131,116,96,151,147,100,155,108,104,159,64,183,179,68,187,76,72,191,171,92,88,175,80,167,163,84,11,252,248,15,240,7,3,244,224,23,19,228,27,236,232,31,192,55,51,196,59,204,200,63,43,220,216,47,208,39,35,212,128,119,115,132,123,140,136,127,107,156,152,111,144,103,99,148,75,188,184,79,176,71,67,180,160,87,83,164,91,172,168,95,22,225,229,18,237,26,30,233,253,10,14,249,6,241,245,2,221,42,46,217,38,209,213,34,54,193,197,50,205,58,62,201,157,106,110,153,102,145,149,98,118,129,133,114,141,122,126,137,86,161,165,82,173,90,94,169,189,74,78,185,70,177,181,66,29,234,238,25,230,17,21,226,246,1,5,242,13,250,254,9,214,33,37,210,45,218,222,41,61,202,206,57,198,49,53,194,150,97,101,146,109,154,158,105,125,138,142,121,134,113,117,130,93,170,174,89,166,81,85,162,182,65,69,178,77,186,190,73,0,248,237,21,199,63,42,210,147,107,126,134,84,172,185,65,59,195,214,46,252,4,17,233,168,80,69,189,111,151,130,122,118,142,155,99,177,73,92,164,229,29,8,240,34,218,207,55,77,181,160,88,138,114,103,159,222,38,51,203,25,225,244,12,236,20,1,249,43,211,198,62,127,135,146,106,184,64,85,173,215,47,58,194,16,232,253,5,68,188,169,81,131,123,110,150,154,98,119,143,93,165,176,72,9,241,228,28,206,54,35,219,161,89,76,180,102,158,139,115,50,202,223,39,245,13,24,224,197,61,40,208,2,250,239,23,86,174,187,67,145,105,124,132,254,6,19,235,57,193,212,44,109,149,128,120,170,82,71,191,179,75,94,166,116,140,153,97,32,216,205,53,231,31,10,242,136,112,101,157,79,183,162,90,27,227,246,14,220,36,49,201,41,209,196,60,238,22,3,251,186,66,87,175,125,133,144,104,18,234,255,7,213,45,56,192,129,121,108,148,70,190,171,83,95,167,178,74,152,96,117,141,204,52,33,217,11,243,230,30,100,156,137,113,163,91,78,182,247,15,26,226,48,200,221,37,0,249,239,22,195,58,44,213,155,98,116,141,88,161,183,78,43,210,196,61,232,17,7,254,176,73,95,166,115,138,156,101,86,175,185,64,149,108,122,131,205,52,34,219,14,247,225,24,125,132,146,107,190,71,81,168,230,31,9,240,37,220,202,51,172,85,67,186,111,150,128,121,55,206,216,33,244,13,27,226,135,126,104,145,68,189,171,82,28,229,243,10,223,38,48,201,250,3,21,236,57,192,214,47,97,152,142,119,162,91,77,180,209,40,62,199,18,235,253,4,74,179,165,92,137,112,102,159,69,188,170,83,134,127,105,144,222,39,49,200,29,228,242,11,110,151,129,120,173,84,66,187,245,12,26,227,54,207,217,32,19,234,252,5,208,41,63,198,136,113,103,158,75,178,164,93,56,193,215,46,251,2,20,237,163,90,76,181,96,153,143,118,233,16,6,255,42,211,197,60,114,139,157,100,177,72,94,167,194,59,45,212,1,248,238,23,89,160,182,79,154,99,117,140,191,70,80,169,124,133,147,106,36,221,203,50,231,30,8,241,148,109,123,130,87,174,184,65,15,246,224,25,204,53,35,218,0,250,233,19,207,53,38,220,131,121,106,144,76,182,165,95,27,225,242,8,212,46,61,199,152,98,113,139,87,173,190,68,54,204,223,37,249,3,16,234,181,79,92,166,122,128,147,105,45,215,196,62,226,24,11,241,174,84,71,189,97,155,136,114,108,150,133,127,163,89,74,176,239,21,6,252,32,218,201,51,119,141,158,100,184,66,81,171,244,14,29,231,59,193,210,40,90,160,179,73,149,111,124,134,217,35,48,202,22,236,255,5,65,187,168,82,142,116,103,157,194,56,43,209,13,247,228,30,216,34,49,203,23,237,254,4,91,161,178,72,148,110,125,135,195,57,42,208,12,246,229,31,64,186,169,83,143,117,102,156,238,20,7,253,33,219,200,50,109,151,132,126,162,88,75,177,245,15,28,230,58,192,211,41,118,140,159,101,185,67,80,170,180,78,93,167,123,129,146,104,55,205,222,36,248,2,17,235,175,85,70,188,96,154,137,115,44,214,197,63,227,25,10,240,130,120,107,145,77,183,164,94,1,251,232,18,206,52,39,221,153,99,112,138,86,172,191,69,26,224,243,9,213,47,60,198,0,251,235,16,203,48,32,219,139,112,96,155,64,187,171,80,11,240,224,27,192,59,43,208,128,123,107,144,75,176,160,91,22,237,253,6,221,38,54,205,157,102,118,141,86,173,189,70,29,230,246,13,214,45,61,198,150,109,125,134,93,166,182,77,44,215,199,60,231,28,12,247,167,92,76,183,108,151,135,124,39,220,204,55,236,23,7,252,172,87,71,188,103,156,140,119,58,193,209,42,241,10,26,225,177,74,90,161,122,129,145,106,49,202,218,33,250,1,17,234,186,65,81,170,113,138,154,97,88,163,179,72,147,104,120,131,211,40,56,195,24,227,243,8,83,168,184,67,152,99,115,136,216,35,51,200,19,232,248,3,78,181,165,94,133,126,110,149,197,62,46,213,14,245,229,30,69,190,174,85,142,117,101,158,206,53,37,222,5,254,238,21,116,143,159,100,191,68,84,175,255,4,20,239,52,207,223,36,127,132,148,111,180,79,95,164,244,15,31,228,63,196,212,47,98,153,137,114,169,82,66,185,233,18,2,249,34,217,201,50,105,146,130,121,162,89,73,178,226,25,9,242,41,210,194,57,0,252,229,25,215,43,50,206,179,79,86,170,100,152,129,125,123,135,158,98,172,80,73,181,200,52,45,209,31,227,250,6,246,10,19,239,33,221,196,56,69,185,160,92,146,110,119,139,141,113,104,148,90,166,191,67,62,194,219,39,233,21,12,240,241,13,20,232,38,218,195,63,66,190,167,91,149,105,112,140,138,118,111,147,93,161,184,68,57,197,220,32,238,18,11,247,7,251,226,30,208,44,53,201,180,72,81,173,99,159,134,122,124,128,153,101,171,87,78,178,207,51,42,214,24,228,253,1,255,3,26,230,40,212,205,49,76,176,169,85,155,103,126,130,132,120,97,157,83,175,182,74,55,203,210,46,224,28,5,249,9,245,236,16,222,34,59,199,186,70,95,163,109,145,136,116,114,142,151,107,165,89,64,188,193,61,36,216,22,234,243,15,14,242,235,23,217,37,60,192,189,65,88,164,106,150,143,115,117,137,144,108,162,94,71,187,198,58,35,223,17,237,244,8,248,4,29,225,47,211,202,54,75,183,174,82,156,96,121,133,131,127,102,154,84,168,177,77,48,204,213,41,231,27,2,254,0,253,231,26,211,46,52,201,187,70,92,161,104,149,143,114,107,150,140,113,184,69,95,162,208,45,55,202,3,254,228,25,214,43,49,204,5,248,226,31,109,144,138,119,190,67,89,164,189,64,90,167,110,147,137,116,6,251,225,28,213,40,50,207,177,76,86,171,98,159,133,120,10,247,237,16,217,36,62,195,218,39,61,192,9,244,238,19,97,156,134,123,178,79,85,168,103,154,128,125,180,73,83,174,220,33,59,198,15,242,232,21,12,241,235,22,223,34,56,197,183,74,80,173,100,153,131,126,127,130,152,101,172,81,75,182,196,57,35,222,23,234,240,13,20,233,243,14,199,58,32,221,175,82,72,181,124,129,155,102,169,84,78,179,122,135,157,96,18,239,245,8,193,60,38,219,194,63,37,216,17,236,246,11,121,132,158,99,170,87,77,176,206,51,41,212,29,224,250,7,117,136,146,111,166,91,65,188,165,88,66,191,118,139,145,108,30,227,249,4,205,48,42,215,24,229,255,2,203,54,44,209,163,94,68,185,112,141,151,106,115,142,148,105,160,93,71,186,200,53,47,210,27,230,252,1,0,254,225,31,223,33,62,192,163,93,66,188,124,130,157,99,91,165,186,68,132,122,101,155,248,6,25,231,39,217,198,56,182,72,87,169,105,151,136,118,21,235,244,10,202,52,43,213,237,19,12,242,50,204,211,45,78,176,175,81,145,111,112,142,113,143,144,110,174,80,79,177,210,44,51,205,13,243,236,18,42,212,203,53,245,11,20,234,137,119,104,150,86,168,183,73,199,57,38,216,24,230,249,7,100,154,133,123,187,69,90,164,156,98,125,131,67,189,162,92,63,193,222,32,224,30,1,255,226,28,3,253,61,195,220,34,65,191,160,94,158,96,127,129,185,71,88,166,102,152,135,121,26,228,251,5,197,59,36,218,84,170,181,75,139,117,106,148,247,9,22,232,40,214,201,55,15,241,238,16,208,46,49,207,172,82,77,179,115,141,146,108,147,109,114,140,76,178,173,83,48,206,209,47,239,17,14,240,200,54,41,215,23,233,246,8,107,149,138,116,180,74,85,171,37,219,196,58,250,4,27,229,134,120,103,153,89,167,184,70,126,128,159,97,161,95,64,190,221,35,60,194,2,252,227,29,0,255,227,28,219,36,56,199,171,84,72,183,112,143,147,108,75,180,168,87,144,111,115,140,224,31,3,252,59,196,216,39,150,105,117,138,77,178,174,81,61,194,222,33,230,25,5,250,221,34,62,193,6,249,229,26,118,137,149,106,173,82,78,177,49,206,210,45,234,21,9,246,154,101,121,134,65,190,162,93,122,133,153,102,161,94,66,189,209,46,50,205,10,245,233,22,167,88,68,187,124,131,159,96,12,243,239,16,215,40,52,203,236,19,15,240,55,200,212,43,71,184,164,91,156,99,127,128,98,157,129,126,185,70,90,165,201,54,42,213,18,237,241,14,41,214,202,53,242,13,17,238,130,125,97,158,89,166,186,69,244,11,23,232,47,208,204,51,95,160,188,67,132,123,103,152,191,64,92,163,100,155,135,120,20,235,247,8,207,48,44,211,83,172,176,79,136,119,107,148,248,7,27,228,35,220,192,63,24,231,251,4,195,60,32,223,179,76,80,175,104,151,139,116,197,58,38,217,30,225,253,2,110,145,141,114,181,74,86,169,142,113,109,146,85,170,182,73,37,218,198,57,254,1,29,226}; - - -/* - ** this function is used to create galois_mult_8_table[65536] -*/ - -/*int galois_create_mult_tables(int w) - - int j, x, y, logx; - - if (w >= 14) return -1; - - if (galois_mult_tables[w] != NULL) return 0; - galois_mult_tables[w] = (int *) malloc(sizeof(int) * nw[w] * nw[w]); - if (galois_mult_tables[w] == NULL) return -1; - - galois_div_tables[w] = (int *) malloc(sizeof(int) * nw[w] * nw[w]); - if (galois_div_tables[w] == NULL) { - free(galois_mult_tables[w]); - galois_mult_tables[w] = NULL; - return -1; - } - if (galois_log_tables[w] == NULL) { - if (galois_create_log_tables(w) < 0) { - free(galois_mult_tables[w]); - free(galois_div_tables[w]); - galois_mult_tables[w] = NULL; - galois_div_tables[w] = NULL; - return -1; - } - } - - j = 0; - galois_mult_tables[w][j] = 0; - galois_div_tables[w][j] = -1; - j++; - for (y = 1; y < nw[w]; y++) { - galois_mult_tables[w][j] = 0; - galois_div_tables[w][j] = 0; - j++; - } - - for (x = 1; x < nw[w]; x++) { - galois_mult_tables[w][j] = 0; - galois_div_tables[w][j] = -1; - j++; - logx = galois_log_tables[w][x]; - for (y = 1; y < nw[w]; y++) { - galois_mult_tables[w][j] = galois_ilog_tables[w][logx+galois_log_tables[w][y]]; - galois_div_tables[w][j] = galois_ilog_tables[w][logx-galois_log_tables[w][y]]; - j++; - } - } - FILE * fp; - fp = fopen("/home/lixiang/zt_hash/table_result.txt","a"); - int i = 0; - fprintf(fp, "mult_tables:\n"); - for(i = 0; i < nw[w]*nw[w]; i++) - { - fprintf(fp, "%d,", galois_mult_tables[w][i]); - } - fprintf(fp, "\ndiv_tables:\n"); - for(i = 0; i < nw[w]*nw[w]; i++) - { - fprintf(fp, "%d,", galois_div_tables[w][i]); - } - - fprintf(fp, "\nlog_tables:\n"); - for(i = 0; i < nw[w]; i++) - { - fprintf(fp, "%d,", galois_log_tables[w][i]); - } - - fprintf(fp, "\nilog_tables:\n"); - for(i = 0; i < nw[w]*3; i++) - { - fprintf(fp, "%d,", galois_ilog_tables[w][i]); - } - fclose(fp); - return 0; -} -*/ - -#define galois_multtable_8_multiply(x,y) galois_mult_8_table[((x)<<8)|(y)] -/* -inline unsigned char galois_multtable_8_multiply(unsigned char x, unsigned char y) -{ - int index = (x<<8) | y; - return galois_mult_8_table[index]; -} -*/ - -/*int galois_destroy_mult_tables(int w) -{ - return 0; -}*/ - -static inline void zt_hash_arymul(struct zt_state_t * a, struct zt_state_t* b) -{ - struct zt_state_t tmp; - tmp.val=a->val; - a->matrix[0] = galois_multtable_8_multiply(tmp.matrix[0],b->matrix[0])^galois_multtable_8_multiply(tmp.matrix[1],b->matrix[2]); - a->matrix[1] = galois_multtable_8_multiply(tmp.matrix[0],b->matrix[1])^galois_multtable_8_multiply(tmp.matrix[1],b->matrix[3]); - a->matrix[2] = galois_multtable_8_multiply(tmp.matrix[2],b->matrix[0])^galois_multtable_8_multiply(tmp.matrix[3],b->matrix[2]); - a->matrix[3] = galois_multtable_8_multiply(tmp.matrix[2],b->matrix[1])^galois_multtable_8_multiply(tmp.matrix[3],b->matrix[3]); -} - -/* - ** this function is used to create the table[4][256] -*/ -/*void convert(int number, unsigned char * ret) -{ - int i,a[8]; - unsigned char tmp[4]; - for(i = 0; i < 8; i++) - { - a[i] = number%2; - number = number/2; - } - if(a[7] == 0) - { - ret[0] = 2; - ret[1] = 1; - ret[2] = 1; - ret[3] = 0; - } - else - { - ret[0] = 2; - ret[1] = 3; - ret[2] = 1; - ret[3] = 1; - } - for(i = 6; i >= 0; i--) - { - if(a[i] == 0) - { - tmp[0] = 2; - tmp[1] = 1; - tmp[2] = 1; - tmp[3] = 0; - - } - else - { - tmp[0] = 2; - tmp[1] = 3; - tmp[2] = 1; - tmp[3] = 1; - } - zt_hash_arymul(ret, tmp); - } -}*/ - - -/* - ** this function is used to create table[4][256] - */ -/*void zt_hash_create_table() -{ - unsigned char ret[4]={0}; - int i = 0; - FILE * fp; - fp = fopen("/home/lixiang/zt_hash/table.txt","a"); - //galois_create_mult_tables(8); //it should not be a comment - for(i = 0; i < 256; i++) - { - convert(i, ret); - table[i].matrix[0] = ret[0]; - table[i].matrix[1] = ret[1]; - table[i].matrix[2] = ret[2]; - table[i].matrix[3] = ret[3]; - fprintf(fp, "{%d,%d,%d,%d},", table[i].matrix[0], - table[i].matrix[1], - table[i].matrix[2], - table[i].matrix[3]); - } -} - -void zt_hash_destroy_table() -{ - int i = 0; - for(i = 0; i < 33; i++) - { - galois_destroy_mult_tables(i); - } -}*/ - - -static inline void zt_hash(struct zt_state_t* array, unsigned char c) -{ - zt_hash_arymul(array, (struct zt_state_t *)(zt_multi_table[c])); -} - -unsigned char ZT_INIT_VAL[4]={1,0,0,1}; -void zt_hash_initial(struct zt_state_t* zt_val) -{ - zt_val->matrix[0] = 1; - zt_val->matrix[1] = 0; - zt_val->matrix[2] = 0; - zt_val->matrix[3] = 1; -} - -int zt_hash_code(struct zt_state_t* zt_val) -{ - int ret; - int tmp[4]; - tmp[0] = zt_val->matrix[0]; - tmp[1] = zt_val->matrix[1]<<8; - tmp[2] = zt_val->matrix[2]<<16; - tmp[3] = zt_val->matrix[3]<<24; - ret = tmp[0]^tmp[1]^tmp[2]^tmp[3]; - return ret; -} - - diff --git a/src/version.map b/src/version.map deleted file mode 100644 index 1087558..0000000 --- a/src/version.map +++ /dev/null @@ -1,15 +0,0 @@ -VERS_3.0{ -global: - extern "C" { - *MAAT_FRAME_VERSION_*; - *Maat_*; - *bool_matcher_*; - *SFH_*; - *GIE_*; -#for test - *my_scandir*; - *md5_file*; - *system_cmd_*; - }; -local: *; -}; \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt deleted file mode 100644 index acc2c2c..0000000 --- a/test/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -#add_executable(maat_demo maat_demo.cpp) -#target_link_libraries(maat_demo maat_frame_shared) - -add_executable(test_igraph test_igraph.cpp) -target_link_libraries(test_igraph igraph-static) - -add_executable(test_maatframe test_maatframe.cpp) -target_link_libraries(test_maatframe maat_frame_shared gtest) - -add_executable(perf_test_maatframe perf_test_maatframe.cpp) -target_link_libraries(perf_test_maatframe maat_frame_shared gtest) - -configure_file(table_info.conf table_info.conf COPYONLY) -configure_file(tsg_tableinfo.conf tsg_tableinfo.conf COPYONLY) -configure_file(file_test_tableinfo.conf file_test_tableinfo.conf COPYONLY) -configure_file(maat_json.json maat_json.json COPYONLY) -configure_file(reset_redis4maat.sh reset_redis4maat.sh COPYONLY) -file(COPY conf DESTINATION ./) -file(COPY rule DESTINATION ./) -file(COPY testdata DESTINATION ./) -file(COPY testdata_uni2ascii DESTINATION ./) -file(COPY test_streamfiles DESTINATION ./) -file(COPY ntcrule DESTINATION ./) -file(COPY tsgrule DESTINATION ./) -file(COPY json_update DESTINATION ./) \ No newline at end of file diff --git a/test/conf/This folder is used to set rulescan scan para.txt b/test/conf/This folder is used to set rulescan scan para.txt deleted file mode 100644 index e69de29..0000000 diff --git a/test/conf/config.txt b/test/conf/config.txt deleted file mode 100644 index 5a2112e..0000000 --- a/test/conf/config.txt +++ /dev/null @@ -1,30 +0,0 @@ -[URL50] -group_num = {2} -group_algor_0 = {AC} -group_algor_1 = {KRF} -group_len_0 = {11} -krf_scale_bits = {5} -[URL100] -group_num = {2} -group_algor_0 = {AC} -group_algor_1 = {KRF} -group_len_0 = {11} -krf_scale_bits = {4} -[URL200] -group_num = {2} -group_algor_0 = {AC} -group_algor_1 = {KRF} -group_len_0 = {11} -krf_scale_bits = {3} -[URL500] -group_num = {2} -group_algor_0 = {AC} -group_algor_1 = {KRF} -group_len_0 = {11} -krf_scale_bits = {3} -[URL1000] -group_num = {2} -group_algor_0 = {AC} -group_algor_1 = {KRF} -group_len_0 = {10} -krf_scale_bits = {5} diff --git a/test/file_test_tableinfo.conf b/test/file_test_tableinfo.conf deleted file mode 100644 index 271b999..0000000 --- a/test/file_test_tableinfo.conf +++ /dev/null @@ -1,27 +0,0 @@ -#each collumn seperate with '\t' -#id (0~65535) -#name string -#type one of ip,expr,expr_plus,digest,intval,compile or plugin -#src_charset one of GBK,BIG5,UNICODE,UTF8 -#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' -#do_merege yes or no -#cross cache 0~max -#quickswitch quickon or quick off -#id name type src_charset dst_charset do_merge cross_cache quickswitch -0 NTC_COMPILE compile UTF8 UTF8 no 0 -0 WHITE_LIST_COMPILE compile UTF8 UTF8 no 0 -1 NTC_GROUP2GROUP group2group -- -2 NTC_GROUP2COMPILE group2compile -- -3 NTC_UNIVERSAL_IP ip UTF8 UTF8 no 0 -4 NTC_UNIVERSAL_PROTO_TYPE intval UTF8 UTF8 no 0 -5 WHITE_LIST_IP ip UTF8 UTF8 no 0 -7 NTC_HTTP_URL expr UTF8 UTF8 yes 0 quickoff -7 WHITE_LIST_DOMAIN expr UTF8 UTF8 yes 0 quickoff -8 NTC_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff -8 NTC_HTTP_RES_HDR expr_plus UTF8 UTF8 yes 0 quickoff -9 NTC_HTTP_REQ_BODY expr UTF8 UTF8/GBK/BIG5/UNICODE yes 0 quickoff -9 NTC_HTTP_RES_BODY expr UTF8 UTF8/GBK/BIG5/UNICODE yes 0 quickoff -11 NTC_MAIL_HDR expr_plus UTF8 UTF8/GBK yes 0 quickoff -12 NTC_MAIL_BODY expr_plus UTF8 UTF8/GBK yes 0 quickoff -13 NTC_FTP_URL expr UTF8 UTF8 yes 0 quickoff -14 NTC_FTP_CONTENT expr UTF8 UTF8 yes 0 quickoff diff --git a/test/file_test_tableinfo.conf.bak b/test/file_test_tableinfo.conf.bak deleted file mode 100644 index 2266965..0000000 --- a/test/file_test_tableinfo.conf.bak +++ /dev/null @@ -1,27 +0,0 @@ -#each collumn seperate with '\t' -#id (0~65535) -#name string -#type one of ip,expr,expr_plus,digest,intval,compile or plugin -#src_charset one of GBK,BIG5,UNICODE,UTF8 -#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' -#do_merege yes or no -#cross cache 0~max -#quickswitch quickon or quick off -#id name type src_charset dst_charset do_merge cross_cache quickswitch -0 NTC_COMPILE compile UTF8 UTF8 no 0 -0 WHITE_LIST_COMPILE compile UTF8 UTF8 no 0 -1 NTC_GROUP2GROUP group2group -- -2 NTC_GROUP2COMPILE group2compile -- -3 NTC_UNIVERSAL_IP ip UTF8 UTF8 no 0 -3 NTC_UNIVERSAL_PROTO_TYPE intval UTF8 UTF8 no 0 -5 WHITE_LIST_IP ip UTF8 UTF8 no 0 -7 NTC_HTTP_URL expr UTF8 UTF8 yes 0 quickoff -7 WHITE_LIST_DOMAIN expr UTF8 UTF8 yes 0 quickoff -8 NTC_HTTP_REQ_HDR expr_plus UTF8 UTF8 yes 0 quickoff -8 NTC_HTTP_RES_HDR expr_plus UTF8 UTF8 yes 0 quickoff -9 NTC_HTTP_REQ_BODY expr UTF8 UTF8/GBK/BIG5/UNICODE yes 0 quickoff -9 NTC_HTTP_RES_BODY expr UTF8 UTF8/GBK/BIG5/UNICODE yes 0 quickoff -11 NTC_MAIL_HDR expr_plus UTF8 UTF8/GBK yes 0 quickoff -12 NTC_MAIL_BODY expr_plus UTF8 UTF8/GBK yes 0 quickoff -13 NTC_FTP_URL expr UTF8 UTF8 yes 0 quickoff -14 NTC_FTP_CONTENT expr UTF8 UTF8 yes 0 quickoff diff --git a/test/json_update/corrupted.json b/test/json_update/corrupted.json deleted file mode 100644 index 66db4d1..0000000 --- a/test/json_update/corrupted.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "compile_table": "COMPILE", - "group_table": "GROUP", - "rules": [ - { - "compile_id": 1 - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "Untitled", - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "hello&world", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - } - ] -} diff --git a/test/json_update/new.json b/test/json_update/new.json deleted file mode 100644 index cfb5238..0000000 --- a/test/json_update/new.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "compile_table": "COMPILE", - "group2compile_table": "GROUP2COMPILE", - "group2group_table": "GROUP2GROUP", - "rules": [ - { - "compile_id": 2, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "MESA&Maat", - "expr_type": "and", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - } - ] -} diff --git a/test/json_update/old.json b/test/json_update/old.json deleted file mode 100644 index 27f5f44..0000000 --- a/test/json_update/old.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "compile_table": "COMPILE", - "group2compile_table": "GROUP2COMPILE", - "group2group_table": "GROUP2GROUP", - "rules": [ - { - "compile_id": 1, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "Untitled", - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "string", - "table_content": { - "keywords": "hello&world", - "expr_type": "and", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - } - ] -} diff --git a/test/maat_demo.cpp b/test/maat_demo.cpp deleted file mode 100644 index 40aed8c..0000000 --- a/test/maat_demo.cpp +++ /dev/null @@ -1,1223 +0,0 @@ -#include "Maat_rule.h" -#include "stream_fuzzy_hash.h" -#include "Maat_command.h" -#include -#include -#include -#include //inet_addr -#include //inet_addr -#include //inet_addr -#include -#include //fstat -#include -#include -#include -#include -#include //fstat -#include //fstat -#include -#include -const char* test_maat_redis_ip="127.0.0.1"; -unsigned short test_maat_redis_port=6379; -const char* json_path="./maat_json.json"; -const char* ful_cfg_dir="./rule/full/index/"; -const char* inc_cfg_dir="./rule/inc/index/"; -#define WAIT_FOR_EFFECTIVE_US 2*1000*1000 -extern int my_scandir(const char *dir, struct dirent ***namelist, - int(*filter)(const struct dirent *), - int(*compar)(const void *, const void *)); -void Maat_read_entry_start_cb(int update_type,void* u_para) -{ - return; -} -void Maat_read_entry_cb(int table_id,const char* table_line,void* u_para) -{ - char ip_str[16]={0}; - int entry_id=-1,seq=-1; - unsigned int ip_uint=0; - int is_valid=0; - unsigned int local_ip_nr=16820416;//192.168.0.1 - sscanf(table_line,"%d\t%s\t%d\t%d",&seq,ip_str,&entry_id,&is_valid); - inet_pton(AF_INET,ip_str,&ip_uint); - if(local_ip_nr==ip_uint) - { - if(is_valid==1) - { - //printf("Load entry id %d success.\n",entry_id); - } - else - { - //printf("Offload entry id %d success.\n",entry_id); - } - } - return; -} -void Maat_read_entry_finish_cb(void* u_para) -{ - Maat_feather_t feather=u_para; - long long version=0; - int ret=0,is_last_updating_table=0; - ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version)); - if(ret!=0) - { - assert(0); - } - ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table)); - if(ret!=0) - { - assert(0); - } - - //printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table); - - return; -} -void print_maat_ret(int ret) -{ - switch(ret) - { - case -1: - printf("scan error.\n"); - break; - case -2: - printf("hit current region,but not hit compile rule.\n"); - break; - case 0: - printf("nothing hit\n"); - break; - default://>0 - printf("hit %d rules\n",ret); - break; - } - return; -} -const char* print_maat_result(struct Maat_rule_t* result,int ret) -{ - static char buff[1024]={0}; - int i=0,j=0; - switch(ret) - { - case -1: - snprintf(buff,sizeof(buff),"ret=%d,scan error.",ret); - break; - case -2: - snprintf(buff,sizeof(buff),"ret=%d,hit current region,but not hit compile rule.",ret); - break; - case 0: - snprintf(buff,sizeof(buff),"ret=0,nothing hit."); - break; - default://>0 - j=snprintf(buff,sizeof(buff),"hit %d rules, hit ruleid=",ret); - for(i=0;iservice_defined,' '); - if(p!=NULL) - { - printf("Test service define escape Success.\n"); - } - else - { - printf("Test service define escape Failed.\n"); - } - -} - -int test_string_full_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int ret=0; - int table_id=0; - struct Maat_rule_t result[4]; - int found_pos[4]; - const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - - ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), - result,found_pos, 4, - mid, 0); - printf("Full String Scan:%s\n",print_maat_result(result,ret)); - return ret; -} -int test_unescape_string_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int ret=0; - int table_id=0; - struct Maat_rule_t result[4]; - int found_pos[4]; - const char* scan_data="Batman\\:Take me Home.Superman/:Fine,stay with me."; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - - ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), - result,found_pos, 4, - mid, 0); - printf("Unescape String Scan:%s\n",print_maat_result(result,ret)); - - return ret; -} -int test_intval_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int table_id=0,ret=0; - int scan_val=2015; - struct Maat_rule_t result[4]; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.",table_name); - - } - else - { - ret=Maat_scan_intval(feather, table_id, scan_val, result,4,mid, 0); - printf("Intval Scan:%s\n",print_maat_result(result,ret)); - } - return ret; -} -int test_str_stream_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int table_id=0,ret=0; - struct Maat_rule_t result[4]; - const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); - stream_para_t sp=Maat_stream_scan_string_start(feather,table_id,0); - int detail_ret=0; - if(sp==NULL) - { - printf("stream scan start failed.\n"); - return -1; - } - ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,"www.cyberessays.com", strlen("www.cyberessays.com") - ,result,4,hit_detail,10 - ,&detail_ret,mid); - ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data, strlen(scan_data) - ,result,4,hit_detail,10 - ,&detail_ret,mid); - Maat_stream_scan_string_end(&sp); - free(hit_detail); - printf("Stream String Scan:%s\n",print_maat_result(result,ret)); - return ret; -} -int test_ipv4_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int table_id=0,ret=0; - struct Maat_rule_t result[4]; - struct ipaddr ipv4_addr; - struct stream_tuple4_v4 v4_addr; - ipv4_addr.addrtype=ADDR_TYPE_IPV4; - inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); - v4_addr.source=htons(50001); - inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); - v4_addr.dest=htons(80); - ipv4_addr.v4=&v4_addr; - - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - - } - else - { - ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,result,4, mid,0); - test_service_define_escape(result+0); - printf("IPv4 addr Scan:%s\n",print_maat_result(result,ret)); - } - return ret; -} -int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int table_id=0,ret=0; - struct Maat_rule_t result[4]; - struct ipaddr ipv6_addr; - struct stream_tuple4_v6 v6_addr; - - ipv6_addr.addrtype=ADDR_TYPE_IPV6; - inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); - v6_addr.source=htons(50001); - inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); - v6_addr.dest=htons(80); - ipv6_addr.v6=&v6_addr; - - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - - } - else - { - //for improving performance. - Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); - ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0); - printf("IPv6 addr Scan:%s\n",print_maat_result(result,ret)); - - if(ret!=-2) - { - printf("ipv6 scan result:%d ,shoulde be -2.\n",ret); - - } - } - return ret; -} -int test_digest_scan(Maat_feather_t feather,const char* table_name,const char* file_name,scan_status_t* mid) -{ - int table_id=0,ret=0; - struct stat digest_fstat; - unsigned long long read_size=0,scan_offset=0; - char digest_test_buff[4096]={0}; - - struct Maat_rule_t result[4]; - stream_para_t sp=NULL; - table_id=Maat_table_register(feather, table_name); - if(table_id<0) - { - printf("registe table %s error.\n",table_name); - return 0; - } - ret=stat(file_name,&digest_fstat); - if(ret!=0) - { - printf("fstat %s error.\n",file_name); - return 0; - } - FILE* fp=fopen(file_name,"r"); - if(fp!=NULL) - { - sp=Maat_stream_scan_digest_start(feather, table_id, digest_fstat.st_size, 0); - while(0==feof(fp)) - { - read_size=fread(digest_test_buff,1,sizeof(digest_test_buff),fp); - ret=Maat_stream_scan_digest(&sp, digest_test_buff, read_size, scan_offset, result,4,mid); - scan_offset+=read_size; - if(ret>0) - { - printf("Digest Scan:%s\n",print_maat_result(result,ret)); - - } - } - fclose(fp); - } - else - { - printf("fopen %s error.\n",file_name); - } - Maat_stream_scan_digest_end(&sp); - return ret; -} -int test_plugin_table(Maat_feather_t feather,const char* table_name, - Maat_start_callback_t *start,Maat_update_callback_t *update,Maat_finish_callback_t *finish, - void *u_para, - void* logger) -{ - int table_id=0,ret=0; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - } - else - { - ret=Maat_table_callback_register(feather, table_id, - start, - update, - finish, - u_para); - if(ret<0) - { - printf("Maat callback register table %s error.\n",table_name); - } - } - return ret; -} -int test_url_encode(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - const char* url_utf8="www.google.com/?q=C%23%E4%B8%AD%E5%9B%BD"; - const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA"; - int table_id=0,ret=0; - struct Maat_rule_t result[4]; - int found_pos[4]; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.",table_name); - return -1; - } - ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, url_utf8, strlen(url_utf8), - result,found_pos, 4, - mid, 0); - printf("URL encode scan utf8 url: %s\n",print_maat_result(result,ret)); - - ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312), - result,found_pos, 4, - mid, 0); - printf("URL encode scan gb2312 url: %s\n",print_maat_result(result,ret)); - - return 0; -} -int test_unicode_esc(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - const char* test_data_dir="./testdata_uni2ascii"; - struct dirent **namelist; - FILE* fp=NULL; - char file_path[256]={0}; - char buff[4096]; - size_t read_len=0; - int table_id=0,ret=0; - struct Maat_rule_t result[4]; - stream_para_t sp=NULL; - int found_pos[4]; - int n=0,i=0; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed in function %s.\n",table_name,__FUNCTION__); - return -1; - } - n = my_scandir(test_data_dir, &namelist, NULL, (int (*)(const void*, const void*))alphasort); - if(n<0) - { - printf("%s open dir %s error.\n",__FUNCTION__,test_data_dir); - return -1; - } - for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) - { - continue; - } - snprintf(file_path,sizeof(file_path),"%s/%s",test_data_dir,namelist[i]->d_name); - fp=fopen(file_path,"rb"); - if(fp==NULL) - { - printf("fopen %s error.\n",file_path);; - continue; - } - printf("%s processing %s\n",__FUNCTION__,file_path); - sp=Maat_stream_scan_string_start(feather,table_id,0); - if(sp==NULL) - { - printf("stream scan start failed.\n"); - continue; - } - read_len=fread(buff,1,sizeof(buff),fp); - while(read_len>0) - { - - ret=Maat_stream_scan_string(&sp,CHARSET_NONE,buff,read_len - ,result,found_pos,4,mid); - read_len=fread(buff,1,sizeof(buff),fp); - if(ret>0) - { - printf("UNI2ASCII file %s,%s\n",file_path,print_maat_result(result,ret)); - } - } - Maat_stream_scan_string_end(&sp); - fclose(fp); - - } - for(i=0;i0) - { - printf("Should not hit without setting district.\n"); - return -1; - } - ret=Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_DISTRICT,region_name,strlen(region_name)); - if(ret<0) - { - printf("set MAAT_SET_SCAN_DISTRICT failed.\n"); - return -1; - } - ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), - result,found_pos, 4, - mid, 0); - if(ret>0) - { - printf("Hit expr_plus rule %d.\n",result[0].config_id); - } - else - { - printf("Test expr_plus failed.\n"); - } - return ret; - -} -int test_string_similar_scan(Maat_feather_t feather,const char* table_name,scan_status_t* mid) -{ - int ret=0; - int table_id=0; - struct Maat_rule_t result[4]; - const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765"; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - - ret=Maat_similar_scan_string(feather, table_id, scan_data, strlen(scan_data), - result, 4, - mid, 0); - printf("Similar String Scan:%s\n",print_maat_result(result,ret)); - return ret; -} - -int test_offset_str_scan_with_chunk(Maat_feather_t feather,const char* table_name,int chunk_size) -{ - int table_id=0,ret=0; - int read_size=0,pass_flag=0; - struct Maat_rule_t result[4]; - scan_status_t mid=NULL; - //const char* fn="./testdata/mesa_logo.jpg"; - const char* fn="./testdata/mesa_logo.jpg"; - FILE*fp=fopen(fn,"r"); - if(fp==NULL) - { - printf("%s open %s failed.\n",__FUNCTION__, fn); - return -1; - } - char scan_data[chunk_size]; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); - stream_para_t sp=Maat_stream_scan_string_start(feather,table_id,0); - int detail_ret=0; - if(sp==NULL) - { - printf("stream scan start failed.\n"); - return -1; - } - while(0==feof(fp)) - { - read_size=fread(scan_data,1,sizeof(scan_data),fp); - ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size - ,result,4,hit_detail,10 - ,&detail_ret,&mid); - if(ret>0) - { - printf("Test offset string Scan chunk=%d Success. String Scan:%s\n",chunk_size,print_maat_result(result,ret)); - pass_flag=1; - break; - } - } - Maat_stream_scan_string_end(&sp); - free(hit_detail); - fclose(fp); - if(!pass_flag) - { - printf("Test offset string Scan Failed.\n"); - } - Maat_clean_status(&mid); - return ret; -} - -void test_offset_str_scan(Maat_feather_t feather,const char* table_name) -{ - test_offset_str_scan_with_chunk(feather,table_name,64); - test_offset_str_scan_with_chunk(feather,table_name,1460); - return; -} -void test_compile_accept_tags(Maat_feather_t feather) -{ - int ret1=0, ret2=0; - int table_id=0; - scan_status_t mid=NULL; - struct Maat_rule_t result[4]; - const char* should_hit="string bbb should hit"; - const char* should_not_hit="string aaa should not hit"; - const char* table_name="HTTP_URL"; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return; - } - - ret1=Maat_full_scan_string(feather, table_id,CHARSET_GBK, should_not_hit, strlen(should_not_hit), - result,NULL, 4, - &mid, 0); - ret2=Maat_full_scan_string(feather, table_id,CHARSET_GBK, should_hit, strlen(should_hit), - result,NULL, 4, - &mid, 0); - if(ret1<=0&&ret2>0) - { - printf("Test compile accept tags success.\n"); - } - else - { - printf("Test compile accept tags failed.\n"); - } - Maat_clean_status(&mid); - return; -} - -void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) -{ - char status[32]={0}; - int entry_id=-1,seq=-1; - int is_valid=0; - sscanf(table_line,"%d\t%s\t%d\t%d",&seq,status,&entry_id,&is_valid); - printf("Test plugin accept tags loading %d %s.\n",seq, status); - return; -} - - -void test_plugin_accept_tags(Maat_feather_t feather) -{ - int table_id=0,ret=0; - const char* table_name="TEST_EFFECTIVE_RANGE_TABLE"; - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - } - else - { - ret=Maat_table_callback_register(feather, table_id, - NULL, - accept_tags_entry_cb, - NULL, - NULL); - if(ret<0) - { - printf("Maat callback register table %s error.\n",table_name); - } - } - return; -} -void test_longer_service_define(Maat_feather_t feather, struct Maat_rule_t* rule) -{ - int ret=0; - char* buff=(char*)malloc(sizeof(char)*rule->serv_def_len); - ret=Maat_read_rule(feather, rule,MAAT_RULE_SERV_DEFINE, buff, rule->serv_def_len); - if(ret==rule->serv_def_len) - { - printf("Test read longer service define Success.\n"); - } - else - { - printf("Test read longer service define Failed.\n"); - } - assert(ret==rule->serv_def_len); - free(buff); -} -int test_table_conjunction(Maat_feather_t feather,const char* table_name,const char* conj_table_name,scan_status_t* mid) -{ - int ret=0; - int table_id=0,conj_table_id=0; - struct Maat_rule_t result[4]; - int found_pos[4]; - const char* scan_data="soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; - - table_id=Maat_table_register(feather,table_name); - if(table_id==-1) - { - printf("Database table %s register failed.\n",table_name); - return -1; - } - conj_table_id=Maat_table_register(feather,conj_table_name); - assert(conj_table_id==table_id); - ret=Maat_full_scan_string(feather, conj_table_id,CHARSET_GBK, scan_data, strlen(scan_data), - result,found_pos, 4, - mid, 0); - if(ret>=2) - { - printf("Table conjunction success %s\n",print_maat_result(result,ret)); - test_longer_service_define(feather, result); - } - return 0; -} -#define TEST_CMD_LINE_NUM 200*1000 -void test_set_cmd_line(Maat_feather_t feather) -{ - struct Maat_cmd_line **p_line=(struct Maat_cmd_line **)calloc(sizeof(struct Maat_cmd_line *), TEST_CMD_LINE_NUM); - struct Maat_cmd_line *line_rule=(struct Maat_cmd_line *)calloc(sizeof(struct Maat_cmd_line), TEST_CMD_LINE_NUM); - int i=0; - const char* line="1\t192.168.0.1\t4444444444\t1"; - memset(&line_rule,0,sizeof(line_rule)); - for(i=0;i