From 6ea9ad9413b83c2d33c0160dcb76a539e2e4875e Mon Sep 17 00:00:00 2001 From: zhangshuai Date: Mon, 19 Aug 2024 09:30:35 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E8=B0=83=E6=95=B4=20pcap=20explore=20?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/controller/ApplicationController.java | 6 - .../app/service/IApplicationService.java | 2 - .../service/impl/ApplicationServiceImpl.java | 84 -------------- .../runner/controller/PcapController.java | 6 + .../module/runner/service/IPcapService.java | 3 + .../runner/service/impl/PcapServiceImpl.java | 108 ++++++++++++++++++ 6 files changed, 117 insertions(+), 92 deletions(-) diff --git a/src/main/java/net/geedge/asw/module/app/controller/ApplicationController.java b/src/main/java/net/geedge/asw/module/app/controller/ApplicationController.java index 2cb127e..ef7fd13 100644 --- a/src/main/java/net/geedge/asw/module/app/controller/ApplicationController.java +++ b/src/main/java/net/geedge/asw/module/app/controller/ApplicationController.java @@ -123,10 +123,4 @@ public class ApplicationController { return R.ok(); } - @GetMapping("/explore") - public R explore(@RequestParam String workspaceId, @RequestParam String pcapIds) { - String discoverUrl = applicationService.generateKibanaDiscoverUrl(workspaceId, pcapIds); - return R.ok().putData("url", discoverUrl); - } - } diff --git a/src/main/java/net/geedge/asw/module/app/service/IApplicationService.java b/src/main/java/net/geedge/asw/module/app/service/IApplicationService.java index 0de8c3d..4de8fab 100644 --- a/src/main/java/net/geedge/asw/module/app/service/IApplicationService.java +++ b/src/main/java/net/geedge/asw/module/app/service/IApplicationService.java @@ -25,6 +25,4 @@ public interface IApplicationService extends IService{ void restore(String id, String version); - String generateKibanaDiscoverUrl(String workspaceId, String pcapIds); - } diff --git a/src/main/java/net/geedge/asw/module/app/service/impl/ApplicationServiceImpl.java b/src/main/java/net/geedge/asw/module/app/service/impl/ApplicationServiceImpl.java index a197f00..f3c7d8a 100644 --- a/src/main/java/net/geedge/asw/module/app/service/impl/ApplicationServiceImpl.java +++ b/src/main/java/net/geedge/asw/module/app/service/impl/ApplicationServiceImpl.java @@ -166,88 +166,4 @@ public class ApplicationServiceImpl extends ServiceImpl pcapIdList = T.StrUtil.split(pcapIds, ",").stream().filter(s -> T.StrUtil.isNotEmpty(s)).collect(Collectors.toList()); - List pcapList = pcapService.list(new LambdaQueryWrapper().in(PcapEntity::getId, pcapIdList)); - T.VerifyUtil.is(pcapList).notEmpty(RCode.SYS_RECORD_NOT_FOUND); - - // index name - String indexName = String.format("workspace-%s-*", workspace.getName()); - - SaTokenInfo tokenInfo = StpUtil.getTokenInfo(); - String token = tokenInfo.getTokenValue(); - - JSONObject index = kibanaClient.findIndexPattern(token, indexName); - JSONArray savedObjects = index.getJSONArray("saved_objects"); - - // check if index exists - boolean indexExists = savedObjects.stream() - .filter(obj -> { - JSONObject attributes = ((JSONObject) obj).getJSONObject("attributes"); - if (T.ObjectUtil.isEmpty(attributes)) return false; - String title = attributes.getString("title"); - return T.StrUtil.equals(indexName, title); - }) - .findFirst() - .isPresent(); - if (log.isDebugEnabled()) { - log.debug("[generateKibanaDiscoverUrl] [idnex-pattern: {}] [exists: {}]", indexName, indexExists); - } - - // create index - if (T.BooleanUtil.negate(indexExists)) { - JSONObject attributes = new JSONObject(); - attributes.put("title", indexName); - - JSONObject body = new JSONObject(); - body.put("attributes", attributes); - kibanaClient.saveIndexPattern(token, workspaceId, body); - } - - // build url - String baseUrl = UrlBuilder.ofHttp(kibanaUrl) - .addPath("/app/data-explorer/discover") - .addQuery("jwt", token) - .toString(); - - // build query param - String param1 = String.format("_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'%s',view:discover))", workspaceId); - String param2 = "_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))"; - - String filter = pcapList.stream() - .map(PcapEntity::getId) - .map(pcapId -> "\"" + pcapId + "\"") - .collect(Collectors.joining("|", "pcap.id: (", ")")); - String param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s'))", filter); - - String query = String.format("?%s&%s&%s", param1, param2, param3); - String kibanaDiscoverUrl = baseUrl + "#" + query; - - if (log.isDebugEnabled()) { - log.debug("[generateKibanaDiscoverUrl] [url: {}]", kibanaDiscoverUrl); - } - return kibanaDiscoverUrl; - } } diff --git a/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java b/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java index f7ac873..c6caf0e 100644 --- a/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java +++ b/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java @@ -168,4 +168,10 @@ public class PcapController { pcapService.unparse2session(ids); return R.ok(); } + + @GetMapping("/explore") + public R explore(@RequestParam String workspaceId, @RequestParam String pcapIds, @RequestParam(required = false) String protocol, @RequestParam(required = false) String streamId) { + String discoverUrl = pcapService.generateKibanaDiscoverUrl(workspaceId, pcapIds, protocol, streamId); + return R.ok().putData("url", discoverUrl); + } } \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java b/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java index de99c95..07c8870 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java +++ b/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.IService; import net.geedge.asw.module.runner.entity.PcapEntity; import org.springframework.core.io.Resource; +import org.springframework.web.bind.annotation.RequestParam; import java.util.Map; @@ -22,4 +23,6 @@ public interface IPcapService extends IService{ void parse2session(String... ids); void unparse2session(String[] ids); + + String generateKibanaDiscoverUrl(String workspaceId, String pcapIds, String protocol, String streamId); } diff --git a/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java b/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java index 4f9949a..52c5c07 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java +++ b/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java @@ -1,7 +1,12 @@ package net.geedge.asw.module.runner.service.impl; +import cn.dev33.satoken.stp.SaTokenInfo; import cn.dev33.satoken.stp.StpUtil; +import cn.hutool.core.net.URLEncodeUtil; +import cn.hutool.core.net.url.UrlBuilder; import cn.hutool.log.Log; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -14,6 +19,7 @@ import net.geedge.asw.module.app.entity.ApplicationEntity; import net.geedge.asw.module.app.entity.PackageEntity; import net.geedge.asw.module.app.service.IApplicationService; import net.geedge.asw.module.app.service.IPackageService; +import net.geedge.asw.module.feign.client.KibanaClient; import net.geedge.asw.module.runner.dao.PcapDao; import net.geedge.asw.module.runner.entity.JobEntity; import net.geedge.asw.module.runner.entity.PcapEntity; @@ -78,6 +84,12 @@ public class PcapServiceImpl extends ServiceImpl implements @Autowired private IWorkspaceService workspaceService; + @Value("${kibana.url:127.0.0.1:5601}") + private String kibanaUrl; + + @jakarta.annotation.Resource + private KibanaClient kibanaClient; + @Override public PcapEntity queryInfo(String id) { PcapEntity pcap = this.getById(id); @@ -265,6 +277,102 @@ public class PcapServiceImpl extends ServiceImpl implements } } + /** + * 1. 根据 workspace_name 查询 index-pattern 是否存在 + * 2. 不存在则创建索引 + * + * 维护格式示例： + * { + * "type": "index-pattern", + * "id": "workspace_id", + * "attributes": { + * "title": "workspace-{workspace_name}-*" + * } + * } + * @param workspaceId + * @param pcapIds + * @return kibana discover url + */ + @Override + public String generateKibanaDiscoverUrl(String workspaceId, String pcapIds, String protocol, String streamId) { + // verify + WorkspaceEntity workspace = workspaceService.getById(workspaceId); + T.VerifyUtil.is(workspace).notNull(RCode.SYS_RECORD_NOT_FOUND); + + List pcapIdList = T.StrUtil.split(pcapIds, ",").stream().filter(s -> T.StrUtil.isNotEmpty(s)).collect(Collectors.toList()); + List pcapList = this.list(new LambdaQueryWrapper().in(PcapEntity::getId, pcapIdList)); + T.VerifyUtil.is(pcapList).notEmpty(RCode.SYS_RECORD_NOT_FOUND); + + // index name + String indexName = String.format("workspace-%s-*", workspace.getName()); + + SaTokenInfo tokenInfo = StpUtil.getTokenInfo(); + String token = tokenInfo.getTokenValue(); + + JSONObject index = kibanaClient.findIndexPattern(token, indexName); + JSONArray savedObjects = index.getJSONArray("saved_objects"); + + // check if index exists + boolean indexExists = savedObjects.stream() + .filter(obj -> { + JSONObject attributes = ((JSONObject) obj).getJSONObject("attributes"); + if (T.ObjectUtil.isEmpty(attributes)) return false; + String title = attributes.getString("title"); + return T.StrUtil.equals(indexName, title); + }) + .findFirst() + .isPresent(); + if (log.isDebugEnabled()) { + log.debug("[generateKibanaDiscoverUrl] [idnex-pattern: {}] [exists: {}]", indexName, indexExists); + } + + // create index + if (T.BooleanUtil.negate(indexExists)) { + JSONObject attributes = new JSONObject(); + attributes.put("title", indexName); + + JSONObject body = new JSONObject(); + body.put("attributes", attributes); + kibanaClient.saveIndexPattern(token, workspaceId, body); + } + + // build url + String baseUrl = UrlBuilder.ofHttp(kibanaUrl) + .addPath("/app/data-explorer/discover") + .addQuery("jwt", token) + .toString(); + + // build query param + String param1 = String.format("_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'%s',view:discover))", workspaceId); + String param2 = "_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))"; + + String filter = pcapList.stream() + .map(PcapEntity::getId) + .map(pcapId -> "\"" + pcapId + "\"") + .collect(Collectors.joining("|", "pcap.id: (", ")")); + String param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s'))", filter); + + if (T.StrUtil.isNotEmpty(protocol)){ + String condition = T.StrUtil.concat(true, "proto:", protocol); + param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s && %s'))", filter, condition); + if (T.StrUtil.isNotEmpty(streamId)){ + condition = T.StrUtil.concat(true, "pcap.", protocol, "_stream:", streamId); + param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s && %s'))", filter, condition); + } + // 处理 空格 & + param3 = URLEncodeUtil.encode(param3); + param3 = param3.replaceAll("&", "%26"); + } + + String query = String.format("?%s&%s&%s", param1, param2, param3); + String kibanaDiscoverUrl = baseUrl + "#" + query; + + if (log.isDebugEnabled()) { + log.debug("[generateKibanaDiscoverUrl] [url: {}]", kibanaDiscoverUrl); + } + return kibanaDiscoverUrl; + } + /** * calculate Parse Thread Timeout *