gfwleak/appsketch-works-asw-controller
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 调整 pcap 文件解析 index_name= workspace-{workspace.name}-{pcap.md5}

Browse Source
This commit is contained in:
shizhendong
2024-08-02 18:27:22 +08:00
parent af0d0e55ca
commit a837a160f9
1 changed files with 19 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/net/geedge/asw/module/runner/util/PcapParserThread.java
View File

@@ -17,6 +17,7 @@ import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.BulkResponse;
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.CreateIndexRequest;
import org.opensearch.client.opensearch.indices.DeleteIndexRequest;
import org.opensearch.client.opensearch.indices.ExistsRequest; import org.opensearch.client.opensearch.indices.ExistsRequest;
import org.opensearch.client.opensearch.indices.IndexSettings; import org.opensearch.client.opensearch.indices.IndexSettings;
@@ -152,8 +153,12 @@ public class PcapParserThread implements Runnable {
* @param jsonArray * @param jsonArray
*/ */
private void uploadToOpenSearch(JSONArray jsonArray) { private void uploadToOpenSearch(JSONArray jsonArray) {
String pcapPath = pcapEntity.getPath();
String md5Hex = T.DigestUtil.md5Hex(T.FileUtil.file(pcapPath));
String workspaceName = pcapEntity.getWorkspace().getName(); String workspaceName = pcapEntity.getWorkspace().getName();
String indexName = String.format("workspace-%s", workspaceName); String indexName = String.format("workspace-%s-%s", workspaceName, md5Hex);
try { try {
// check if index exists // check if index exists
boolean indexExists = openSearchClient.indices() boolean indexExists = openSearchClient.indices()
@@ -162,23 +167,25 @@ public class PcapParserThread implements Runnable {
if (log.isDebugEnabled()) { if (log.isDebugEnabled()) {
log.debug("[uploadToOpenSearch] [index: {}] [exists: {}]", indexName, indexExists); log.debug("[uploadToOpenSearch] [index: {}] [exists: {}]", indexName, indexExists);
} }
// if index not exists, create index with default settings // if index exists, delete
if (!indexExists) { if (indexExists) {
openSearchClient.indices().create( openSearchClient.indices().delete(new DeleteIndexRequest.Builder().index(indexName).build());
new CreateIndexRequest.Builder() log.debug("[uploadToOpenSearch] [index: {}] [deleted]", indexName);
.index(indexName)
.settings(new IndexSettings.Builder().build())
.build()
);
log.debug("[uploadToOpenSearch] [index: {}] [created]", indexName);
} }
// create index with default settings
openSearchClient.indices().create(
new CreateIndexRequest.Builder()
.index(indexName)
.settings(new IndexSettings.Builder().build())
.build()
);
// upload data in bulk // upload data in bulk
BulkRequest.Builder br = new BulkRequest.Builder(); BulkRequest.Builder br = new BulkRequest.Builder();
String pcapId = pcapEntity.getId();
for (int i = 0; i < jsonArray.size(); i++) { for (int i = 0; i < jsonArray.size(); i++) {
JSONObject jsonObject = (JSONObject) jsonArray.get(i); JSONObject jsonObject = (JSONObject) jsonArray.get(i);
String id = pcapId + "-" + String.valueOf(i); String id = String.valueOf(i);
br.operations(op -> op.index( br.operations(op -> op.index(
idx -> idx.index(indexName) idx -> idx.index(indexName)
.id(id) .id(id)