gfwleak/appsketch-works-asw-controller
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 调整 pcap explore 接口

Browse Source
This commit is contained in:
zhangshuai
2024-08-19 09:30:35 +08:00
parent 6bb44b101d
commit 6ea9ad9413
6 changed files with 117 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/net/geedge/asw/module/app/controller/ApplicationController.java
View File

@@ -123,10 +123,4 @@ public class ApplicationController {
return R.ok();
}
@GetMapping("/explore")
public R explore(@RequestParam String workspaceId, @RequestParam String pcapIds) {
String discoverUrl = applicationService.generateKibanaDiscoverUrl(workspaceId, pcapIds);
return R.ok().putData("url", discoverUrl);
}
}

2
src/main/java/net/geedge/asw/module/app/service/IApplicationService.java
View File

@@ -25,6 +25,4 @@ public interface IApplicationService extends IService<ApplicationEntity>{
void restore(String id, String version);
String generateKibanaDiscoverUrl(String workspaceId, String pcapIds);
}

84
src/main/java/net/geedge/asw/module/app/service/impl/ApplicationServiceImpl.java
View File

@@ -166,88 +166,4 @@ public class ApplicationServiceImpl extends ServiceImpl<ApplicationDao, Applicat
ApplicationEntity application = T.BeanUtil.toBean(oldApplication, ApplicationEntity.class);
this.updateById(application);
}
/**
* 1. 根据 workspace_name 查询 index-pattern 是否存在
* 2. 不存在则创建索引
*
* 维护格式示例：
* {
* "type": "index-pattern",
* "id": "workspace_id",
* "attributes": {
* "title": "workspace-{workspace_name}-*"
* }
* }
* @param workspaceId
* @param pcapIds
* @return kibana discover url
*/
@Override
public String generateKibanaDiscoverUrl(String workspaceId, String pcapIds) {
// verify
WorkspaceEntity workspace = workspaceService.getById(workspaceId);
T.VerifyUtil.is(workspace).notNull(RCode.SYS_RECORD_NOT_FOUND);
List<String> pcapIdList = T.StrUtil.split(pcapIds, ",").stream().filter(s -> T.StrUtil.isNotEmpty(s)).collect(Collectors.toList());
List<PcapEntity> pcapList = pcapService.list(new LambdaQueryWrapper<PcapEntity>().in(PcapEntity::getId, pcapIdList));
T.VerifyUtil.is(pcapList).notEmpty(RCode.SYS_RECORD_NOT_FOUND);
// index name
String indexName = String.format("workspace-%s-*", workspace.getName());
SaTokenInfo tokenInfo = StpUtil.getTokenInfo();
String token = tokenInfo.getTokenValue();
JSONObject index = kibanaClient.findIndexPattern(token, indexName);
JSONArray savedObjects = index.getJSONArray("saved_objects");
// check if index exists
boolean indexExists = savedObjects.stream()
.filter(obj -> {
JSONObject attributes = ((JSONObject) obj).getJSONObject("attributes");
if (T.ObjectUtil.isEmpty(attributes)) return false;
String title = attributes.getString("title");
return T.StrUtil.equals(indexName, title);
})
.findFirst()
.isPresent();
if (log.isDebugEnabled()) {
log.debug("[generateKibanaDiscoverUrl] [idnex-pattern: {}] [exists: {}]", indexName, indexExists);
}
// create index
if (T.BooleanUtil.negate(indexExists)) {
JSONObject attributes = new JSONObject();
attributes.put("title", indexName);
JSONObject body = new JSONObject();
body.put("attributes", attributes);
kibanaClient.saveIndexPattern(token, workspaceId, body);
}
// build url
String baseUrl = UrlBuilder.ofHttp(kibanaUrl)
.addPath("/app/data-explorer/discover")
.addQuery("jwt", token)
.toString();
// build query param
String param1 = String.format("_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'%s',view:discover))", workspaceId);
String param2 = "_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))";
String filter = pcapList.stream()
.map(PcapEntity::getId)
.map(pcapId -> "\"" + pcapId + "\"")
.collect(Collectors.joining("|", "pcap.id: (", ")"));
String param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s'))", filter);
String query = String.format("?%s&%s&%s", param1, param2, param3);
String kibanaDiscoverUrl = baseUrl + "#" + query;
if (log.isDebugEnabled()) {
log.debug("[generateKibanaDiscoverUrl] [url: {}]", kibanaDiscoverUrl);
}
return kibanaDiscoverUrl;
}
}

6
src/main/java/net/geedge/asw/module/runner/controller/PcapController.java
View File

@@ -168,4 +168,10 @@ public class PcapController {
pcapService.unparse2session(ids);
return R.ok();
}
@GetMapping("/explore")
public R explore(@RequestParam String workspaceId, @RequestParam String pcapIds, @RequestParam(required = false) String protocol, @RequestParam(required = false) String streamId) {
String discoverUrl = pcapService.generateKibanaDiscoverUrl(workspaceId, pcapIds, protocol, streamId);
return R.ok().putData("url", discoverUrl);
}
}

3
src/main/java/net/geedge/asw/module/runner/service/IPcapService.java
View File

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.asw.module.runner.entity.PcapEntity;
import org.springframework.core.io.Resource;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
@@ -22,4 +23,6 @@ public interface IPcapService extends IService<PcapEntity>{
void parse2session(String... ids);
void unparse2session(String[] ids);
String generateKibanaDiscoverUrl(String workspaceId, String pcapIds, String protocol, String streamId);
}

108
src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java
View File

@@ -1,7 +1,12 @@
package net.geedge.asw.module.runner.service.impl;
import cn.dev33.satoken.stp.SaTokenInfo;
import cn.dev33.satoken.stp.StpUtil;
import cn.hutool.core.net.URLEncodeUtil;
import cn.hutool.core.net.url.UrlBuilder;
import cn.hutool.log.Log;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@@ -14,6 +19,7 @@ import net.geedge.asw.module.app.entity.ApplicationEntity;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.app.service.IApplicationService;
import net.geedge.asw.module.app.service.IPackageService;
import net.geedge.asw.module.feign.client.KibanaClient;
import net.geedge.asw.module.runner.dao.PcapDao;
import net.geedge.asw.module.runner.entity.JobEntity;
import net.geedge.asw.module.runner.entity.PcapEntity;
@@ -78,6 +84,12 @@ public class PcapServiceImpl extends ServiceImpl<PcapDao, PcapEntity> implements
@Autowired
private IWorkspaceService workspaceService;
@Value("${kibana.url:127.0.0.1:5601}")
private String kibanaUrl;
@jakarta.annotation.Resource
private KibanaClient kibanaClient;
@Override
public PcapEntity queryInfo(String id) {
PcapEntity pcap = this.getById(id);
@@ -265,6 +277,102 @@ public class PcapServiceImpl extends ServiceImpl<PcapDao, PcapEntity> implements
}
}
/**
* 1. 根据 workspace_name 查询 index-pattern 是否存在
* 2. 不存在则创建索引
*
* 维护格式示例：
* {
* "type": "index-pattern",
* "id": "workspace_id",
* "attributes": {
* "title": "workspace-{workspace_name}-*"
* }
* }
* @param workspaceId
* @param pcapIds
* @return kibana discover url
*/
@Override
public String generateKibanaDiscoverUrl(String workspaceId, String pcapIds, String protocol, String streamId) {
// verify
WorkspaceEntity workspace = workspaceService.getById(workspaceId);
T.VerifyUtil.is(workspace).notNull(RCode.SYS_RECORD_NOT_FOUND);
List<String> pcapIdList = T.StrUtil.split(pcapIds, ",").stream().filter(s -> T.StrUtil.isNotEmpty(s)).collect(Collectors.toList());
List<PcapEntity> pcapList = this.list(new LambdaQueryWrapper<PcapEntity>().in(PcapEntity::getId, pcapIdList));
T.VerifyUtil.is(pcapList).notEmpty(RCode.SYS_RECORD_NOT_FOUND);
// index name
String indexName = String.format("workspace-%s-*", workspace.getName());
SaTokenInfo tokenInfo = StpUtil.getTokenInfo();
String token = tokenInfo.getTokenValue();
JSONObject index = kibanaClient.findIndexPattern(token, indexName);
JSONArray savedObjects = index.getJSONArray("saved_objects");
// check if index exists
boolean indexExists = savedObjects.stream()
.filter(obj -> {
JSONObject attributes = ((JSONObject) obj).getJSONObject("attributes");
if (T.ObjectUtil.isEmpty(attributes)) return false;
String title = attributes.getString("title");
return T.StrUtil.equals(indexName, title);
})
.findFirst()
.isPresent();
if (log.isDebugEnabled()) {
log.debug("[generateKibanaDiscoverUrl] [idnex-pattern: {}] [exists: {}]", indexName, indexExists);
}
// create index
if (T.BooleanUtil.negate(indexExists)) {
JSONObject attributes = new JSONObject();
attributes.put("title", indexName);
JSONObject body = new JSONObject();
body.put("attributes", attributes);
kibanaClient.saveIndexPattern(token, workspaceId, body);
}
// build url
String baseUrl = UrlBuilder.ofHttp(kibanaUrl)
.addPath("/app/data-explorer/discover")
.addQuery("jwt", token)
.toString();
// build query param
String param1 = String.format("_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'%s',view:discover))", workspaceId);
String param2 = "_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))";
String filter = pcapList.stream()
.map(PcapEntity::getId)
.map(pcapId -> "\"" + pcapId + "\"")
.collect(Collectors.joining("|", "pcap.id: (", ")"));
String param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s'))", filter);
if (T.StrUtil.isNotEmpty(protocol)){
String condition = T.StrUtil.concat(true, "proto:", protocol);
param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s && %s'))", filter, condition);
if (T.StrUtil.isNotEmpty(streamId)){
condition = T.StrUtil.concat(true, "pcap.", protocol, "_stream:", streamId);
param3 = String.format("_q=(filters:!(),query:(language:lucene,query:'%s && %s'))", filter, condition);
}
// 处理 空格 &
param3 = URLEncodeUtil.encode(param3);
param3 = param3.replaceAll("&", "%26");
}
String query = String.format("?%s&%s&%s", param1, param2, param3);
String kibanaDiscoverUrl = baseUrl + "#" + query;
if (log.isDebugEnabled()) {
log.debug("[generateKibanaDiscoverUrl] [url: {}]", kibanaDiscoverUrl);
}
return kibanaDiscoverUrl;
}
/**
* calculate Parse Thread Timeout
*