gfwleak/galaxy-tsg-olap-flume-dynamic-complement
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: gfwleak:master
Branches Tags
gfwleak:master gfwleak:20.11-rc1
...
pull from: gfwleak:20.11-rc1
Branches Tags
gfwleak:master gfwleak:20.11-rc1

7 Commits
master ... 20.11-rc1

Author SHA1 Message Date
戚岱杰
6c8557a568 Delete workspace.xml 2021-04-23 08:59:37 +00:00
戚岱杰
3e5982e8d7 Delete vcs.xml 2021-04-23 08:59:33 +00:00
戚岱杰
1df0822fa6 Delete misc.xml 2021-04-23 08:59:30 +00:00
戚岱杰
686ed6ca3e Delete encodings.xml 2021-04-23 08:59:26 +00:00
戚岱杰
db0110387b Delete compiler.xml 2021-04-23 08:59:22 +00:00
戚岱杰
955ad00927 Delete Project_Default.xml 2021-04-23 08:59:16 +00:00
qidaijie
10bf6b71e9 更新20.11-rc1 2020-10-28 17:16:37 +08:00
15 changed files with 534 additions and 904 deletions
Expand all files Collapse all files

15
.idea/compiler.xml generated
View File

@@ -1,15 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="FlumeDynamicInterceptor" />
<module name="FlumeRadiusOnOffInterceptor" />
<module name="FlumeSubscriberInterceptor" />
</profile>
</annotationProcessing>
</component>
</project>

8
.idea/encodings.xml generated
View File

@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/FlumeDynamicInterceptor" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/FlumeRadiusOnOffInterceptor" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/FlumeSubscriberInterceptor" charset="UTF-8" />
</component>
</project>

36
.idea/inspectionProfiles/Project_Default.xml generated
View File

@@ -1,36 +0,0 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="JavaDoc" enabled="true" level="WARNING" enabled_by_default="true">
<option name="TOP_LEVEL_CLASS_OPTIONS">
<value>
<option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
<option name="REQUIRED_TAGS" value="" />
</value>
</option>
<option name="INNER_CLASS_OPTIONS">
<value>
<option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
<option name="REQUIRED_TAGS" value="" />
</value>
</option>
<option name="METHOD_OPTIONS">
<value>
<option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
<option name="REQUIRED_TAGS" value="@return@param@throws or @exception" />
</value>
</option>
<option name="FIELD_OPTIONS">
<value>
<option name="ACCESS_JAVADOC_REQUIRED_FOR" value="none" />
<option name="REQUIRED_TAGS" value="" />
</value>
</option>
<option name="IGNORE_DEPRECATED" value="false" />
<option name="IGNORE_JAVADOC_PERIOD" value="true" />
<option name="IGNORE_DUPLICATED_THROWS" value="false" />
<option name="IGNORE_POINT_TO_ITSELF" value="false" />
<option name="myAdditionalJavadocTags" value="date" />
</inspection_tool>
</profile>
</component>

14
.idea/misc.xml generated
View File

@@ -1,14 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

6
.idea/vcs.xml generated
View File

@@ -1,6 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

658
.idea/workspace.xml generated
View File

@@ -1,658 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="7803fbe0-0492-4e54-9d86-e0390d08ce3a" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java" beforeDir="false" afterPath="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java" beforeDir="false" afterPath="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/target/" />
<ignored path="$PROJECT_DIR$/FlumeDynamicInterceptor/target/" />
<ignored path="$PROJECT_DIR$/FlumeSubscriberInterceptor/target/" />
<ignored path="$PROJECT_DIR$/FlumeRadiusOnOffInterceptor/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="TRACKING_ENABLED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FUSProjectUsageTrigger">
<session id="-2092497702">
<usages-collector id="statistics.lifecycle.project">
<counts>
<entry key="project.closed" value="1" />
<entry key="project.open.time.1" value="1" />
<entry key="project.opened" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.open">
<counts>
<entry key="java" value="8" />
<entry key="properties" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.open">
<counts>
<entry key="JAVA" value="8" />
<entry key="Properties" value="1" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.extensions.edit">
<counts>
<entry key="java" value="32" />
</counts>
</usages-collector>
<usages-collector id="statistics.file.types.edit">
<counts>
<entry key="JAVA" value="32" />
</counts>
</usages-collector>
</session>
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="FlumeDynamicApp.java" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1395">
<caret line="123" column="31" selection-start-line="123" selection-start-column="31" selection-end-line="123" selection-end-column="31" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="pom.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/pom.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
</file>
<file leaf-file-name="HBaseUtils.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="247">
<caret line="26" column="67" selection-start-line="26" selection-start-column="67" selection-end-line="26" selection-end-column="67" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="HttpClientUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/http/HttpClientUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="704">
<caret line="52" selection-start-line="52" selection-end-line="52" />
<folding>
<element signature="e#1795#1841#0" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="JsonParseUtil.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1505">
<caret line="86" column="25" selection-start-line="86" selection-start-column="25" selection-end-line="86" selection-end-column="25" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="FlowWriteConfigurations.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/FlowWriteConfigurations.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="323">
<caret line="17" column="33" selection-start-line="17" selection-start-column="33" selection-end-line="17" selection-end-column="33" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="FlowWriteConfig.java" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/FlowWriteConfig.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="798">
<caret line="42" column="93" selection-start-line="42" selection-start-column="93" selection-end-line="42" selection-end-column="93" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="service_flow_config.properties" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/properties/service_flow_config.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="171">
<caret line="9" column="19" selection-start-line="9" selection-start-column="19" selection-end-line="9" selection-end-column="19" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/http/HttpClientUtil.java" />
<option value="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java" />
<option value="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/FlowWriteConfigurations.java" />
<option value="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java" />
<option value="$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java" />
</list>
</option>
</component>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="MavenImportPreferences">
<option name="generalSettings">
<MavenGeneralSettings>
<option name="localRepository" value="D:\MAVEN\repo" />
<option name="mavenHome" value="D:/apache-maven-3.5.4" />
<option name="userSettingsFile" value="D:\apache-maven-3.5.4\conf\settings.xml" />
</MavenGeneralSettings>
</option>
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
</component>
<component name="MavenProjectNavigator">
<treeState>
<expand>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="FlumeDynamicInterceptor" type="9519ce18:MavenProjectsStructure$ProjectNode" />
</path>
<path>
<item name="" type="16c1761:MavenProjectsStructure$RootNode" />
<item name="FlumeDynamicInterceptor" type="9519ce18:MavenProjectsStructure$ProjectNode" />
<item name="Lifecycle" type="58874e2:MavenProjectsStructure$LifecycleNode" />
</path>
</expand>
<select />
</treeState>
</component>
<component name="NodePackageJsonFileManager">
<packageJsonPaths />
</component>
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="850" />
<option name="y" value="92" />
<option name="width" value="950" />
<option name="height" value="846" />
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="PackagesPane" />
<pane id="Scope" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="properties" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="common" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
<item name="hbase" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
<item name="http" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
<item name="json" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="java" type="462c0819:PsiDirectoryNode" />
<item name="interceptor" type="462c0819:PsiDirectoryNode" />
<item name="utils" type="462c0819:PsiDirectoryNode" />
<item name="system" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="dynamic_complement" type="b2602c69:ProjectViewProjectNode" />
<item name="dynamic_complement" type="462c0819:PsiDirectoryNode" />
<item name="FlumeDynamicInterceptor" type="462c0819:PsiDirectoryNode" />
<item name="target" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="AndroidView" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="com.android.tools.idea.instantapp.provision.ProvisionBeforeRunTaskProvider.myTimeStamp" value="1591960947841" />
<property name="go.gopath.indexing.explicitly.defined" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="MavenSettings" />
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager">
<configuration default="true" type="Application" factoryName="Application">
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<patterns />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" value="-ea" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" value="%MODULE_WORKING_DIR%" />
<option name="OUTPUT_DIRECTORY" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<properties />
<listeners />
</configuration>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="7803fbe0-0492-4e54-9d86-e0390d08ce3a" name="Default Changelist" comment="" />
<created>1591958573741</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1591958573741</updated>
<workItem from="1591958576333" duration="1762000" />
<workItem from="1591962716014" duration="1355000" />
<workItem from="1592188791616" duration="6660000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="9777000" />
</component>
<component name="ToolWindowManager">
<frame x="-8" y="-8" width="1936" height="1056" extended-state="7" />
<editor active="true" />
<layout>
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="right" id="Palette&#9;" order="3" />
<window_info id="Image Layers" order="2" />
<window_info anchor="bottom" id="Java Enterprise" order="7" />
<window_info anchor="right" id="Capture Analysis" order="3" />
<window_info anchor="bottom" id="Event Log" order="7" side_tool="true" weight="0.25460455" />
<window_info anchor="right" id="Maven Projects" order="3" weight="0.17803837" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Version Control" order="7" />
<window_info anchor="bottom" id="Run" order="2" weight="0.32936078" />
<window_info anchor="bottom" id="Terminal" order="7" />
<window_info id="Capture Tool" order="2" />
<window_info id="Designer" order="2" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25266525" />
<window_info anchor="right" id="Bean Validation" order="3" />
<window_info anchor="right" id="Database" order="3" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info id="UI Designer" order="2" />
<window_info anchor="right" id="Theme Preview" order="3" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Find" order="1" />
</layout>
<layout-to-restore>
<window_info anchor="right" id="Palette" order="3" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="right" id="Capture Analysis" order="4" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="right" id="Theme Preview" order="6" />
<window_info id="Favorites" order="6" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25266525" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Version Control" order="10" />
<window_info id="Image Layers" order="2" />
<window_info id="Designer" order="3" />
<window_info id="UI Designer" order="4" />
<window_info anchor="bottom" id="Find" order="1" />
<window_info anchor="right" id="Bean Validation" order="7" />
<window_info anchor="right" id="Database" order="5" />
<window_info anchor="right" id="Palette&#9;" order="8" />
<window_info anchor="bottom" id="Terminal" order="8" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Event Log" order="9" side_tool="true" />
<window_info id="Capture Tool" order="5" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Java Enterprise" order="11" />
<window_info anchor="right" id="Maven Projects" order="9" visible="true" weight="0.17803837" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
</layout-to-restore>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2736">
<caret line="165" column="30" lean-forward="true" selection-start-line="165" selection-start-column="30" selection-end-line="165" selection-end-column="30" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/pom.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1235">
<caret line="78" column="27" selection-start-line="78" selection-start-column="27" selection-end-line="78" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/http/HttpClientUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="817">
<caret line="52" selection-start-line="52" selection-end-line="52" />
<folding>
<element signature="e#1795#1841#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3534">
<caret line="193" column="28" selection-start-line="193" selection-start-column="28" selection-end-line="193" selection-end-column="28" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/FlowWriteConfigurations.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="323">
<caret line="17" column="33" selection-start-line="17" selection-start-column="33" selection-end-line="17" selection-end-column="33" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/FlowWriteConfig.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="798">
<caret line="42" column="93" selection-start-line="42" selection-start-column="93" selection-end-line="42" selection-end-column="93" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/properties/service_flow_config.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="12" selection-start-line="12" selection-end-line="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="4617">
<caret line="270" column="48" selection-start-line="270" selection-start-column="48" selection-end-line="270" selection-end-column="48" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1235">
<caret line="78" column="27" selection-start-line="78" selection-start-column="27" selection-end-line="78" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/http/HttpClientUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="817">
<caret line="52" selection-start-line="52" selection-end-line="52" />
<folding>
<element signature="e#1795#1841#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3534">
<caret line="193" column="28" selection-start-line="193" selection-start-column="28" selection-end-line="193" selection-end-column="28" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/FlowWriteConfigurations.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="323">
<caret line="17" column="33" lean-forward="true" selection-start-line="17" selection-start-column="33" selection-end-line="17" selection-end-column="33" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/FlowWriteConfig.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="798">
<caret line="42" column="93" lean-forward="true" selection-start-line="42" selection-start-column="93" selection-end-line="42" selection-end-column="93" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/properties/service_flow_config.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="228">
<caret line="12" selection-start-line="12" selection-end-line="12" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/BAKJsonParseUtil.java" />
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/SnowflakeId.java" />
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/FlowWriteConfig.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="798">
<caret line="42" column="93" selection-start-line="42" selection-start-column="93" selection-end-line="42" selection-end-column="93" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/pom.xml">
<provider selected="true" editor-type-id="text-editor" />
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="247">
<caret line="26" column="67" selection-start-line="26" selection-start-column="67" selection-end-line="26" selection-end-column="67" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/http/HttpClientUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="704">
<caret line="52" selection-start-line="52" selection-end-line="52" />
<folding>
<element signature="e#1795#1841#0" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1505">
<caret line="86" column="25" selection-start-line="86" selection-start-column="25" selection-end-line="86" selection-end-column="25" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/properties/service_flow_config.properties">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="171">
<caret line="9" column="19" selection-start-line="9" selection-start-column="19" selection-end-line="9" selection-end-column="19" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/system/FlowWriteConfigurations.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="323">
<caret line="17" column="33" selection-start-line="17" selection-start-column="33" selection-end-line="17" selection-end-column="33" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1395">
<caret line="123" column="31" selection-start-line="123" selection-start-column="31" selection-end-line="123" selection-end-column="31" />
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>

135
FlumeDynamicInterceptor/dependency-reduced-pom.xml
View File

@@ -1,135 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>dynamic_complement</artifactId>
<groupId>com.zdjizhi</groupId>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>FlumeDynamicInterceptor</artifactId>
<build>
<resources>
<resource>
<directory>properties</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer />
<transformer>
<mainClass>com.zdjizhi.flume.interceptor.FlumeDynamicApp</mainClass>
</transformer>
<transformer>
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer>
<resource>META-INF/spring.schemas</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<executions>
<execution>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
<configuration>
<executable>java</executable>
<includeProjectDependencies>true</includeProjectDependencies>
<includePluginDependencies>false</includePluginDependencies>
<classpathScope>compile</classpathScope>
<mainClass>com.zdjizhi.flume.interceptor.FlumeDynamicApp</mainClass>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>nexus</id>
<name>Team Nexus Repository</name>
<url>http://192.168.40.125:8099/content/groups/public</url>
</repository>
<repository>
<id>ebi</id>
<name>www.ebi.ac.uk</name>
<url>http://www.ebi.ac.uk/intact/maven/nexus/content/groups/public/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>flume-ng-sdk</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>flume-ng-configuration</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>flume-ng-auth</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>avro-ipc</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-jmx</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>libthrift</artifactId>
<groupId>org.apache.thrift</groupId>
</exclusion>
<exclusion>
<artifactId>mina-core</artifactId>
<groupId>org.apache.mina</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<hbase.version>2.2.1</hbase.version>
<flume.version>1.9.0</flume.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
</project>

24
FlumeDynamicInterceptor/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>dynamic_complement</artifactId>
<groupId>com.zdjizhi</groupId>
<version>1.0</version>
<version>v3.20.09.22</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -23,6 +23,20 @@
<name>www.ebi.ac.uk</name>
<url>http://www.ebi.ac.uk/intact/maven/nexus/content/groups/public/</url>
</repository>
<repository>
<id>maven-ali</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
</repository>
</repositories>
<properties>
@@ -150,7 +164,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
<version>1.2.70</version>
</dependency>
<!--<dependency>-->
@@ -214,6 +228,12 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.4.0</version>
</dependency>
<!--<dependency>-->
<!--<groupId>org.apache.hadoop</groupId>-->
<!--<artifactId>hadoop-common</artifactId>-->

136
FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java
View File

@@ -2,6 +2,8 @@ package com.zdjizhi.flume.interceptor;
import com.alibaba.fastjson.JSONObject;
import com.google.common.base.Preconditions;
import com.jayway.jsonpath.InvalidPathException;
import com.jayway.jsonpath.JsonPath;
import com.zdjizhi.flume.interceptor.common.FlowWriteConfig;
import com.zdjizhi.flume.interceptor.utils.hbase.HBaseUtils;
import com.zdjizhi.flume.interceptor.utils.json.JsonParseUtil;
@@ -133,18 +135,27 @@ public class FlumeDynamicApp implements Interceptor {
Object name = JsonParseUtil.getValue(object, strings[0]);
String appendToKeyName = strings[1];
String functionName = strings[2];
Object param = null;
if (strings[3] != null) {
param = JsonParseUtil.getValue(object, strings[3]);
}
// Object param = null;
// if (strings[3] != null) {
// param = JsonParseUtil.getValue(object, strings[3]);
// }
String param = strings[3];
switch (functionName) {
case "current_timestamp":
// if (Long.parseLong(JsonParseUtil.getValue(object, appendToKeyName)) == 0L) {
JsonParseUtil.setValue(object, appendToKeyName, getCurrentTime());
// }
break;
case "snowflake_id":
JsonParseUtil.setValue(object, appendToKeyName, formatUtils.getSnowflakeId(uidZookeeperIp, dataCenterIdNum));
break;
case "flattenSpec":
if (name != null && StringUtil.isNotBlank(param)) {
JsonParseUtil.setValue(object, appendToKeyName, flattenSpec(name.toString(), isJsonValue(object, param)));
}
break;
case "geo_ip_detail":
if (name != null && JsonParseUtil.getValue(object, appendToKeyName) == null) {
JsonParseUtil.setValue(object, appendToKeyName, getGeoIpDetail(name.toString()));
@@ -155,6 +166,11 @@ public class FlumeDynamicApp implements Interceptor {
JsonParseUtil.setValue(object, appendToKeyName, getGeoAsn(name.toString()));
}
break;
case "if":
if (param != null) {
JsonParseUtil.setValue(object, appendToKeyName, condition(object, param));
}
break;
case "get_value":
if (name != null) {
JsonParseUtil.setValue(object, appendToKeyName, name.toString());
@@ -172,11 +188,7 @@ public class FlumeDynamicApp implements Interceptor {
break;
case "decode_of_base64":
if (name != null) {
if (param != null) {
JsonParseUtil.setValue(object, appendToKeyName, Encodes.decodeBase64String(name.toString(), param.toString()));
} else {
JsonParseUtil.setValue(object, appendToKeyName, Encodes.decodeBase64String(name.toString(), FlowWriteConfig.MAIL_DEFAULT_CHARSET));
}
JsonParseUtil.setValue(object, appendToKeyName, decodeBase64(name.toString(), isJsonValue(object, param)));
}
break;
case "sub_domain":
@@ -208,14 +220,20 @@ public class FlumeDynamicApp implements Interceptor {
* @param url
* @return 顶级域名
*/
private String replaceGetTopDomain(String url) {
return url.replaceAll("InternetDomainName\\{name=", "").replaceAll("\\}", "");
private static String replaceGetTopDomain(String url) {
// return url.replaceAll("InternetDomainName\\{name=", "").replaceAll("\\}", "");
try {
return FormatUtils.getTopPrivateDomain(url);
} catch (StringIndexOutOfBoundsException outException) {
logger.error("解析顶级域名异常,异常域名:" + url, outException);
return "";
}
}
/**
* 生成当前时间戳的操作
*/
private int getCurrentTime() {
private static int getCurrentTime() {
return (int) (System.currentTimeMillis() / 1000);
}
@@ -225,7 +243,7 @@ public class FlumeDynamicApp implements Interceptor {
* @param ip
* @return
*/
private String getGeoIpDetail(String ip) {
private static String getGeoIpDetail(String ip) {
return ipLookup.cityLookupDetail(ip);
}
@@ -235,7 +253,7 @@ public class FlumeDynamicApp implements Interceptor {
* @param ip
* @return
*/
private String getGeoAsn(String ip) {
private static String getGeoAsn(String ip) {
// return ipLookup.asnLookup(ip, true);//v1.0.2-com.zdjizhi.galaxy
return ipLookup.asnLookup(ip);//v1.0.3-com.zdjizhi.galaxy
}
@@ -246,7 +264,7 @@ public class FlumeDynamicApp implements Interceptor {
* @param ip
* @return
*/
private String getGeoIpCountry(String ip) {
private static String getGeoIpCountry(String ip) {
return ipLookup.countryLookup(ip);
}
@@ -260,6 +278,94 @@ public class FlumeDynamicApp implements Interceptor {
return HBaseUtils.getAccount(ip, hbaseZookeeper, hbaseTable);
}
/**
* 根据编码解码base64
*
* @param message base64
* @param charset 编码
* @return 解码字符串
*/
private static String decodeBase64(String message, Object charset) {
String result = "";
try {
if (StringUtil.isNotBlank(message)) {
if (charset != null) {
result = Encodes.decodeBase64String(message, charset.toString());
} else {
result = Encodes.decodeBase64String(message, FlowWriteConfig.MAIL_DEFAULT_CHARSET);
}
}
} catch (Exception e) {
logger.error("解析 Base64 异常,异常信息:" + e);
}
return result;
}
/**
* 根据表达式解析json
*
* @param message json
* @param expr 解析表达式
* @return 解析结果
*/
private static String flattenSpec(String message, String expr) {
String flattenResult = "";
try {
ArrayList<String> read = JsonPath.parse(message).read(expr);
flattenResult = read.get(0);
} catch (ClassCastException | InvalidPathException e) {
logger.error("设备标签解析异常，[ " + expr + " ]解析表达式错误", e);
}
return flattenResult;
}
/**
* 判断是否为日志字段,是则返回对应value，否则返回原始字符串
*
* @param object 内存实体类
* @param param 字段名/普通字符串
* @return JSON.Value or String
*/
private static String isJsonValue(Object object, String param) {
if (param.contains(FlowWriteConfig.IS_JSON_KEY_TAG)) {
Object value = JsonParseUtil.getValue(object, param.substring(2));
if (value != null) {
return value.toString();
} else {
return "";
}
} else {
return param;
}
}
/**
* IF函数实现，解析日志构建三目运算
*
* @param object 内存实体类
* @param ifParam 字段名/普通字符串
* @return resultA or resultB or ""
*/
private static String condition(Object object, String ifParam) {
String result = "";
try {
String[] split = ifParam.split(FlowWriteConfig.FORMAT_SPLITTER);
String[] norms = split[0].split(FlowWriteConfig.IF_CONDITION_SPLITTER);
String direction = isJsonValue(object, norms[0]);
if (StringUtil.isNotBlank(direction)) {
if (split.length == FlowWriteConfig.IF_PARAM_LENGTH) {
String resultA = isJsonValue(object, split[1]);
String resultB = isJsonValue(object, split[2]);
result = (Integer.parseInt(direction) == Integer.parseInt(norms[1])) ? resultA : resultB;
}
}
} catch (Exception e) {
logger.error("IF 函数执行异常,异常信息:" + e);
e.printStackTrace();
}
return result;
}
public static class FlumeDynamicAppBuilder implements Interceptor.Builder {
private String schemaHttpUrl;
private String uidZookeeperIp;

5
FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/FlowWriteConfig.java
View File

@@ -7,6 +7,11 @@ import com.zdjizhi.flume.interceptor.utils.system.FlowWriteConfigurations;
* @author Administrator
*/
public class FlowWriteConfig {
public static final String VISIBILITY = "disabled";
public static final int IF_PARAM_LENGTH = 3;
public static final String FORMAT_SPLITTER = ",";
public static final String IS_JSON_KEY_TAG = "$.";
public static final String IF_CONDITION_SPLITTER = "=";
// public static final String SEGMENTATION = ",";

39
FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/json/JsonParseUtil.java
View File

@@ -3,6 +3,8 @@ package com.zdjizhi.flume.interceptor.utils.json;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.jayway.jsonpath.JsonPath;
import com.zdjizhi.flume.interceptor.common.FlowWriteConfig;
import com.zdjizhi.flume.interceptor.utils.http.HttpClientUtil;
import net.sf.cglib.beans.BeanGenerator;
import net.sf.cglib.beans.BeanMap;
@@ -118,17 +120,42 @@ public class JsonParseUtil {
JSONArray fields = (JSONArray) schemaJson.get("fields");
for (Object field : fields) {
String name = JSON.parseObject(field.toString()).get("name").toString();
String type = JSON.parseObject(field.toString()).get("type").toString();
map.put(name, getClassName(type));
String filedStr = field.toString();
if (checkKeepField(filedStr)) {
String name = JsonPath.read(filedStr, "$.name").toString();
String type = JsonPath.read(filedStr, "$.type").toString();
//组合用来生成实体类的map
map.put(name, getClassName(type));
}
}
return map;
}
public static void main(String[] args) {
System.out.println(getMapFromhttp("http://192.168.44.12:9999/metadata/schema/v1/fields/connection_record_log").toString());
}
/**
* 判断字段是否需要保留
*
* @param message 单个field-json
* @return true or false
*/
private static boolean checkKeepField(String message) {
boolean isKeepField = true;
boolean isHiveDoc = JSON.parseObject(message).containsKey("doc");
if (isHiveDoc) {
boolean isHiveVi = JsonPath.read(message, "$.doc").toString().contains("visibility");
if (isHiveVi) {
String visibility = JsonPath.read(message, "$.doc.visibility").toString();
if (FlowWriteConfig.VISIBILITY.equals(visibility)) {
isKeepField = false;
}
}
}
return isKeepField;
}
/**
* 根据http链接获取schema，解析之后返回一个任务列表 (useList toList funcList)
*

317
FlumeDynamicInterceptor/src/test/java/com/zdjizhi/flume/JsonTest.java Normal file
View File

@@ -0,0 +1,317 @@
package com.zdjizhi.flume;
import com.alibaba.fastjson.JSONObject;
import com.jayway.jsonpath.InvalidPathException;
import com.jayway.jsonpath.JsonPath;
import com.zdjizhi.flume.interceptor.FlumeDynamicApp;
import com.zdjizhi.flume.interceptor.common.FlowWriteConfig;
import com.zdjizhi.flume.interceptor.utils.hbase.HBaseUtils;
import com.zdjizhi.flume.interceptor.utils.json.JsonParseUtil;
import com.zdjizhi.utils.Encodes;
import com.zdjizhi.utils.FormatUtils;
import com.zdjizhi.utils.IpLookup;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
import java.util.ArrayList;
import java.util.HashMap;
/**
* @author qidaijie
* @Package com.zdjizhi.flume
* @Description:
* @date 2020/9/229:38
*/
public class JsonTest {
private static Logger logger = Logger.getLogger(JsonTest.class);
private static IpLookup ipLookup;
private static FormatUtils formatUtils;
private static String schemaHttpUrl = "http://192.168.44.12:9999/metadata/schema/v1/fields/connection_record_log";
private static String uidZookeeperIp = "192.168.44.11:2181,192.168.44.14:2181,192.168.44.15:2181";
private static long dataCenterIdNum = 1L;
private static String ipDatPath = "D:\\workerSpace\\K18-Phase2\\tsgSpace\\dat\\";
private static String hbaseZookeeperIp = "192.168.44.11:2181,192.168.44.14:2181,192.168.44.15:2181";
private static String hbaseTableName = "subscriber_info";
private static HashMap<String, Class> map;
private static Object mapObject;
private static ArrayList<String[]> jobList;
public static void initialize() {
map = JsonParseUtil.getMapFromhttp(schemaHttpUrl);
mapObject = JsonParseUtil.generateObject(map);
jobList = JsonParseUtil.getJobListFromHttp(schemaHttpUrl);
//载入工具类
formatUtils = new FormatUtils.Builder(false).build();
//载入定位库
ipLookup = new IpLookup.Builder(false)
/**
* v1.0.2-com.zdjizhi.galaxy
*/
// .loadDataFileV4(ipDatPath + "Kazakhstan.mmdb")
// .loadDataFileV6(ipDatPath + "Kazakhstan.mmdb")
// .loadAsnDataFileV4(ipDatPath + "asn_v4.mmdb")
// .loadAsnDataFileV6(ipDatPath + "asn_v6.mmdb")
/**
* v1.0.3-com.zdjizhi.galaxy
*/
.loadDataFileV4(ipDatPath + "ip_v4.mmdb")
.loadDataFileV6(ipDatPath + "ip_v6.mmdb")
.loadDataFilePrivateV4(ipDatPath + "ip_private_v4.mmdb")
.loadDataFilePrivateV6(ipDatPath + "ip_private_v6.mmdb")
.loadAsnDataFile(ipDatPath + "asn_v4.mmdb")
.loadAsnDataFileV6(ipDatPath + "asn_v6.mmdb")
.build();
}
public static void main(String[] args) {
initialize();
String json = "{\"common_client_ip\":\"192.168.40.2\",\"common_server_ip\":\"192.168.40.1\",\"common_direction\":0,\"mail_attachment_name\":\"dGVzdA==\",\"common_device_tag\":{\"tag_sets\": [{\"tag\": \"data_center\", \"value\": \"北京/朝阳/华严北里/甲22号\"}, {\"tag\": \"isp\", \"value\": \"电信\"}]}}";
System.out.println(dealCommonMessage(json));
}
private static String dealCommonMessage(String message) {
Object object = JSONObject.parseObject(message, mapObject.getClass());
try {
for (String[] strings : jobList) {
Object name = JsonParseUtil.getValue(object, strings[0]);
String appendToKeyName = strings[1];
String functionName = strings[2];
// Object param = null;
// if (strings[3] != null) {
// param = JsonParseUtil.getValue(object, strings[3]);
// }
String param = strings[3];
switch (functionName) {
case "current_timestamp":
// if (Long.parseLong(JsonParseUtil.getValue(object, appendToKeyName)) == 0L) {
JsonParseUtil.setValue(object, appendToKeyName, getCurrentTime());
// }
break;
case "snowflake_id":
JsonParseUtil.setValue(object, appendToKeyName, formatUtils.getSnowflakeId(uidZookeeperIp, dataCenterIdNum));
break;
case "flattenSpec":
if (name != null && StringUtil.isNotBlank(param)) {
JsonParseUtil.setValue(object, appendToKeyName, flattenSpec(name.toString(), isJsonValue(object, param)));
}
break;
case "geo_ip_detail":
if (name != null && JsonParseUtil.getValue(object, appendToKeyName) == null) {
JsonParseUtil.setValue(object, appendToKeyName, getGeoIpDetail(name.toString()));
}
break;
case "geo_asn":
if (name != null && JsonParseUtil.getValue(object, appendToKeyName) == null) {
JsonParseUtil.setValue(object, appendToKeyName, getGeoAsn(name.toString()));
}
break;
case "if":
if (param != null) {
JsonParseUtil.setValue(object, appendToKeyName, condition(object, param));
}
break;
case "get_value":
if (name != null) {
JsonParseUtil.setValue(object, appendToKeyName, name.toString());
}
break;
case "radius_match":
if (name != null) {
JsonParseUtil.setValue(object, appendToKeyName, radiusMatch(name.toString(), hbaseZookeeperIp, hbaseTableName));
}
break;
case "geo_ip_country":
if (name != null && JsonParseUtil.getValue(object, appendToKeyName) == null) {
JsonParseUtil.setValue(object, appendToKeyName, getGeoIpCountry(name.toString()));
}
break;
case "decode_of_base64":
if (name != null) {
JsonParseUtil.setValue(object, appendToKeyName, decodeBase64(name.toString(), isJsonValue(object, param)));
}
break;
case "sub_domain":
if (name != null) {
Object appendTo = JsonParseUtil.getValue(object, appendToKeyName);
if (appendTo == null || StringUtil.isBlank(appendTo.toString())) {
JsonParseUtil.setValue(object, appendToKeyName, replaceGetTopDomain(FormatUtils.getTopPrivateDomain(name.toString())));
}
}
break;
default:
}
}
return JSONObject.toJSONString(object);
} catch (Exception e) {
logger.error("FlumeDynamicApp dealCommonMessage is error===>{" + e + "}<===");
e.printStackTrace();
// return "";
return message;//返回原数据
}
}
/**
* 有host根据host获取域名,有sni通过sni获取域名
* 这里是直接根据传入的获取域名
*
* @param url
* @return 顶级域名
*/
private static String replaceGetTopDomain(String url) {
// return url.replaceAll("InternetDomainName\\{name=", "").replaceAll("\\}", "");
try {
return FormatUtils.getTopPrivateDomain(url);
} catch (StringIndexOutOfBoundsException outException) {
logger.error("解析顶级域名异常,异常域名:" + url, outException);
return "";
}
}
/**
* 生成当前时间戳的操作
*/
private static int getCurrentTime() {
return (int) (System.currentTimeMillis() / 1000);
}
/**
* 根据clientIp获取location信息
*
* @param ip
* @return
*/
private static String getGeoIpDetail(String ip) {
return ipLookup.cityLookupDetail(ip);
}
/**
* 根据ip获取asn信息
*
* @param ip
* @return
*/
private static String getGeoAsn(String ip) {
// return ipLookup.asnLookup(ip, true);//v1.0.2-com.zdjizhi.galaxy
return ipLookup.asnLookup(ip);//v1.0.3-com.zdjizhi.galaxy
}
/**
* 根据ip获取country信息
*
* @param ip
* @return
*/
private static String getGeoIpCountry(String ip) {
return ipLookup.countryLookup(ip);
}
/**
* radius借助hbase补齐
*
* @param ip
* @return
*/
private static String radiusMatch(String ip, String hbaseZookeeper, String hbaseTable) {
return HBaseUtils.getAccount(ip, hbaseZookeeper, hbaseTable);
}
/**
* 根据编码解码base64
*
* @param message base64
* @param charset 编码
* @return 解码字符串
*/
private static String decodeBase64(String message, Object charset) {
String result = "";
try {
if (StringUtil.isNotBlank(message)) {
if (charset != null) {
result = Encodes.decodeBase64String(message, charset.toString());
} else {
result = Encodes.decodeBase64String(message, FlowWriteConfig.MAIL_DEFAULT_CHARSET);
}
}
} catch (Exception e) {
logger.error("解析 Base64 异常,异常信息:" + e);
}
return result;
}
/**
* 根据表达式解析json
*
* @param message json
* @param expr 解析表达式
* @return 解析结果
*/
private static String flattenSpec(String message, String expr) {
String flattenResult = "";
try {
ArrayList<String> read = JsonPath.parse(message).read(expr);
flattenResult = read.get(0);
} catch (ClassCastException | InvalidPathException e) {
logger.error("设备标签解析异常，[ " + expr + " ]解析表达式错误", e);
}
return flattenResult;
}
/**
* 判断是否为日志字段,是则返回对应value，否则返回原始字符串
*
* @param object 内存实体类
* @param param 字段名/普通字符串
* @return JSON.Value or String
*/
private static String isJsonValue(Object object, String param) {
if (param.contains(FlowWriteConfig.IS_JSON_KEY_TAG)) {
Object value = JsonParseUtil.getValue(object, param.substring(2));
if (value != null) {
return value.toString();
} else {
return "";
}
} else {
return param;
}
}
/**
* IF函数实现，解析日志构建三目运算
*
* @param object 内存实体类
* @param ifParam 字段名/普通字符串
* @return resultA or resultB or ""
*/
private static String condition(Object object, String ifParam) {
String result = "";
try {
String[] split = ifParam.split(FlowWriteConfig.FORMAT_SPLITTER);
String[] norms = split[0].split(FlowWriteConfig.IF_CONDITION_SPLITTER);
String direction = isJsonValue(object, norms[0]);
if (StringUtil.isNotBlank(direction)) {
if (split.length == FlowWriteConfig.IF_PARAM_LENGTH) {
String resultA = isJsonValue(object, split[1]);
String resultB = isJsonValue(object, split[2]);
result = (Integer.parseInt(direction) == Integer.parseInt(norms[1])) ? resultA : resultB;
}
}
} catch (Exception e) {
logger.error("IF 函数执行异常,异常信息:" + e);
e.printStackTrace();
}
return result;
}
}

19
FlumeRadiusOnOffInterceptor/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>dynamic_complement</artifactId>
<groupId>com.zdjizhi</groupId>
<version>1.0</version>
<version>v3.20.09.22</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -17,6 +17,19 @@
<name>Team Nexus Repository</name>
<url>http://192.168.40.125:8099/content/groups/public</url>
</repository>
<repository>
<id>maven-ali</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
</repository>
</repositories>
<properties>
@@ -114,7 +127,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
<version>1.2.70</version>
</dependency>
<dependency>
@@ -126,7 +139,7 @@
<dependency>
<groupId>com.zdjizhi</groupId>
<artifactId>galaxy</artifactId>
<version>1.0.2</version>
<version>1.0.3</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>

9
FlumeRadiusOnOffInterceptor/src/main/java/com/zdjizhi/flume/interceptor/common/OnOffConfig.java
View File

@@ -18,15 +18,16 @@ public class OnOffConfig {
*/
public static final int STOP_BILLING = 2;
/**
* 计费请求报文类型
*/
public static final String RADIUS_ACCT_STATUS_TYPE = "radius_acct_status_type";
/**
* 报文类型
*/
public static final String RADIUS_PACKET_TYPE = "radius_packet_type";
/**
* 计费请求报文类型
*/
public static final String RADIUS_ACCT_STATUS_TYPE = "radius_acct_status_type";
/**
* 发送计费请求报文时间戳
*/

17
FlumeSubscriberInterceptor/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>dynamic_complement</artifactId>
<groupId>com.zdjizhi</groupId>
<version>1.0</version>
<version>v3.20.09.22</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -18,6 +18,19 @@
<name>www.ebi.ac.uk</name>
<url>http://www.ebi.ac.uk/intact/maven/nexus/content/groups/public/</url>
</repository>
<repository>
<id>maven-ali</id>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
</repository>
</repositories>
<properties>
@@ -115,7 +128,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
<version>1.2.70</version>
</dependency>
<dependency>