From 4f8084eaf8d64fa51de86fe598805761c973da08 Mon Sep 17 00:00:00 2001 From: shizhendong Date: Tue, 23 Jul 2024 11:37:47 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20ASW-10=20=E6=96=B0=E5=A2=9E=20pcap=20?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../asw/common/config/SpringContextUtils.java | 41 + .../runner/controller/PcapController.java | 58 + .../geedge/asw/module/runner/dao/PcapDao.java | 6 + .../asw/module/runner/entity/PcapEntity.java | 20 + .../module/runner/service/IJobService.java | 3 +- .../module/runner/service/IPcapService.java | 14 +- .../runner/service/impl/JobServiceImpl.java | 19 +- .../runner/service/impl/PcapServiceImpl.java | 161 ++- .../module/runner/util/PcapParserThread.java | 80 ++ .../module/runner/util/SignatureExtract.java | 179 +++ .../asw/module/runner/util/SignatureUtil.java | 1075 +++++++++++++++++ .../resources/db/mapper/runner/JobMapper.xml | 1 - .../resources/db/mapper/runner/PcapMapper.xml | 126 ++ .../db/migration/V1.0.01__INIT_TABLES.sql | 4 +- 14 files changed, 1767 insertions(+), 20 deletions(-) create mode 100644 src/main/java/net/geedge/asw/common/config/SpringContextUtils.java create mode 100644 src/main/java/net/geedge/asw/module/runner/controller/PcapController.java create mode 100644 src/main/java/net/geedge/asw/module/runner/util/PcapParserThread.java create mode 100644 src/main/java/net/geedge/asw/module/runner/util/SignatureExtract.java create mode 100644 src/main/java/net/geedge/asw/module/runner/util/SignatureUtil.java create mode 100644 src/main/resources/db/mapper/runner/PcapMapper.xml diff --git a/src/main/java/net/geedge/asw/common/config/SpringContextUtils.java b/src/main/java/net/geedge/asw/common/config/SpringContextUtils.java new file mode 100644 index 0000000..aa40daf --- /dev/null +++ b/src/main/java/net/geedge/asw/common/config/SpringContextUtils.java @@ -0,0 +1,41 @@ +package net.geedge.asw.common.config; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +/** + * Spring Context 工具类 + */ +@Component +public class SpringContextUtils implements ApplicationContextAware { + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + SpringContextUtils.applicationContext = applicationContext; + } + + public static Object getBean(String name) { + return applicationContext.getBean(name); + } + + public static T getBean(Class requiredType) { + return applicationContext.getBean(requiredType); + } + + public static String getProperty(String key, String defaultValue) { + return applicationContext.getEnvironment().getProperty(key, defaultValue); + } + + public static T getBean(String name, Class requiredType) { + return applicationContext.getBean(name, requiredType); + } + + public static Class getType(String name) { + return applicationContext.getType(name); + } + +} \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java b/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java new file mode 100644 index 0000000..f14e4d2 --- /dev/null +++ b/src/main/java/net/geedge/asw/module/runner/controller/PcapController.java @@ -0,0 +1,58 @@ +package net.geedge.asw.module.runner.controller; + +import cn.hutool.log.Log; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import net.geedge.asw.common.util.R; +import net.geedge.asw.common.util.RCode; +import net.geedge.asw.common.util.T; +import net.geedge.asw.module.runner.entity.PcapEntity; +import net.geedge.asw.module.runner.service.IPcapService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/pcap") +public class PcapController { + + private static final Log log = Log.get(); + + @Autowired + private IPcapService pcapService; + + @GetMapping("/{id}") + public R detail(@PathVariable("id") String id) { + PcapEntity pcapEntity = pcapService.queryInfo(id); + return R.ok().putData("record", pcapEntity); + } + + @GetMapping + public R list(@RequestParam Map params) { + T.VerifyUtil.is(params).notNull() + .and(T.MapUtil.getStr(params, "workspaceId")).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY); + + Page page = pcapService.queryList(params); + return R.ok(page); + } + + @PostMapping + public R add(@RequestParam(value = "file", required = true) MultipartFile file, + @RequestParam(required = false) String tags, + @RequestParam(required = false) String workbookId, + @RequestParam(required = false) String workspaceId) throws IOException { + T.VerifyUtil.is(workspaceId).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY); + + PcapEntity pcapEntity = pcapService.savePcap(file.getResource(), tags, workbookId, workspaceId); + return R.ok().putData("id", pcapEntity.getId()); + } + + @DeleteMapping("/{id}") + public R delete(@PathVariable("id") String id) { + pcapService.deletePcap(id); + return R.ok(); + } + +} \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/dao/PcapDao.java b/src/main/java/net/geedge/asw/module/runner/dao/PcapDao.java index ef1b878..ffd0d17 100644 --- a/src/main/java/net/geedge/asw/module/runner/dao/PcapDao.java +++ b/src/main/java/net/geedge/asw/module/runner/dao/PcapDao.java @@ -1,10 +1,16 @@ package net.geedge.asw.module.runner.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import net.geedge.asw.module.runner.entity.PcapEntity; import org.apache.ibatis.annotations.Mapper; +import java.util.List; +import java.util.Map; + @Mapper public interface PcapDao extends BaseMapper{ + List queryList(Page page, Map params); + } diff --git a/src/main/java/net/geedge/asw/module/runner/entity/PcapEntity.java b/src/main/java/net/geedge/asw/module/runner/entity/PcapEntity.java index 2e610b9..82fc81a 100644 --- a/src/main/java/net/geedge/asw/module/runner/entity/PcapEntity.java +++ b/src/main/java/net/geedge/asw/module/runner/entity/PcapEntity.java @@ -1,9 +1,13 @@ package net.geedge.asw.module.runner.entity; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +import net.geedge.asw.module.app.entity.ApplicationEntity; +import net.geedge.asw.module.app.entity.PackageEntity; @Data @TableName("pcap") @@ -27,4 +31,20 @@ public class PcapEntity { private String createUserId; private String workspaceId; + @TableField(exist = false) + private String jobId; + + @TableField(exist = false) + private ApplicationEntity application; + + @TableField(exist = false) + @JsonProperty(value = "package") + private PackageEntity pkg; + + @TableField(exist = false) + private RunnerEntity runner; + + @TableField(exist = false) + private PlaybookEntity playbook; + } \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/service/IJobService.java b/src/main/java/net/geedge/asw/module/runner/service/IJobService.java index 9564c88..c0341e8 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/IJobService.java +++ b/src/main/java/net/geedge/asw/module/runner/service/IJobService.java @@ -5,7 +5,6 @@ import com.baomidou.mybatisplus.extension.service.IService; import net.geedge.asw.module.runner.entity.JobEntity; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -23,6 +22,6 @@ public interface IJobService extends IService{ void appendTraceLogStrToFile(String jobId, String content) throws RuntimeException; - void updateJobResult(String jobId, String state, MultipartFile pcapFile) throws IOException; + void updateJobResult(String jobId, String state, MultipartFile pcapFile); } diff --git a/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java b/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java index c293035..7b0e72b 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java +++ b/src/main/java/net/geedge/asw/module/runner/service/IPcapService.java @@ -1,12 +1,22 @@ package net.geedge.asw.module.runner.service; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.IService; import net.geedge.asw.module.runner.entity.PcapEntity; +import org.springframework.core.io.Resource; -import java.io.InputStream; +import java.util.Map; public interface IPcapService extends IService{ - void savePcap(String jobId, InputStream inputStream); + PcapEntity queryInfo(String id); + + Page queryList(Map params); + + PcapEntity savePcap(String jobId, Resource fileResource); + + PcapEntity savePcap(Resource fileResource,String... params); + + void deletePcap(String id); } diff --git a/src/main/java/net/geedge/asw/module/runner/service/impl/JobServiceImpl.java b/src/main/java/net/geedge/asw/module/runner/service/impl/JobServiceImpl.java index fbd3cc1..29473d6 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/impl/JobServiceImpl.java +++ b/src/main/java/net/geedge/asw/module/runner/service/impl/JobServiceImpl.java @@ -14,6 +14,7 @@ import net.geedge.asw.module.app.service.IApplicationService; import net.geedge.asw.module.app.service.IPackageService; import net.geedge.asw.module.runner.dao.JobDao; import net.geedge.asw.module.runner.entity.JobEntity; +import net.geedge.asw.module.runner.entity.PcapEntity; import net.geedge.asw.module.runner.entity.PlaybookEntity; import net.geedge.asw.module.runner.entity.RunnerEntity; import net.geedge.asw.module.runner.service.IJobService; @@ -29,7 +30,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Map; @@ -163,19 +163,22 @@ public class JobServiceImpl extends ServiceImpl implements IJ @Override @Transactional(rollbackFor = Exception.class) - public void updateJobResult(String jobId, String state, MultipartFile pcapFile) throws IOException { - // update job status + public void updateJobResult(String jobId, String state, MultipartFile pcapFile) { + String pcapId = T.StrUtil.EMPTY; + // save pcap file + if (T.ObjectUtil.isNotNull(pcapFile)) { + PcapEntity pcapEntity = pcapService.savePcap(jobId, pcapFile.getResource()); + pcapId = pcapEntity.getId(); + } + + // update job status&pcap_id state = T.StrUtil.equals("success", state) ? RunnerConstant.JobStatus.PASSED.getValue() : state; this.update(new LambdaUpdateWrapper() .set(JobEntity::getStatus, state) + .set(T.StrUtil.isNotEmpty(pcapId), JobEntity::getPcapId, pcapId) .set(JobEntity::getEndTimestamp, System.currentTimeMillis()) .eq(JobEntity::getId, jobId) ); - - // save pcap file - if (T.ObjectUtil.isNotNull(pcapFile)) { - pcapService.savePcap(jobId, pcapFile.getInputStream()); - } } } diff --git a/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java b/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java index beb5df3..1791341 100644 --- a/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java +++ b/src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java @@ -1,21 +1,172 @@ package net.geedge.asw.module.runner.service.impl; +import cn.dev33.satoken.stp.StpUtil; +import cn.hutool.log.Log; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import net.geedge.asw.common.util.RCode; +import net.geedge.asw.common.util.T; +import net.geedge.asw.module.app.entity.ApplicationEntity; +import net.geedge.asw.module.app.entity.PackageEntity; +import net.geedge.asw.module.app.service.IApplicationService; +import net.geedge.asw.module.app.service.IPackageService; import net.geedge.asw.module.runner.dao.PcapDao; +import net.geedge.asw.module.runner.entity.JobEntity; import net.geedge.asw.module.runner.entity.PcapEntity; +import net.geedge.asw.module.runner.entity.PlaybookEntity; +import net.geedge.asw.module.runner.entity.RunnerEntity; +import net.geedge.asw.module.runner.service.IJobService; import net.geedge.asw.module.runner.service.IPcapService; +import net.geedge.asw.module.runner.service.IPlaybookService; +import net.geedge.asw.module.runner.service.IRunnerService; +import net.geedge.asw.module.runner.util.PcapParserThread; +import net.geedge.asw.module.runner.util.RunnerConstant; +import net.geedge.asw.module.workbook.service.IWorkbookResourceService; +import net.geedge.asw.module.workbook.util.WorkbookConstant; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; -import java.io.InputStream; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; @Service public class PcapServiceImpl extends ServiceImpl implements IPcapService { + private static final Log log = Log.get(); + + @Autowired + private IJobService jobService; + + @Autowired + private IRunnerService runnerService; + + @Autowired + private IPlaybookService playbookService; + + @Autowired + private IPackageService packageService; + + @Autowired + private IApplicationService applicationService; + + @Autowired + private IWorkbookResourceService workbookResourceService; + + @Override + public PcapEntity queryInfo(String id) { + PcapEntity pcap = this.getById(id); + T.VerifyUtil.is(pcap).notNull(RCode.SYS_RECORD_NOT_FOUND); + + JobEntity job = jobService.getOne(new LambdaQueryWrapper().eq(JobEntity::getPcapId, pcap.getId())); + if (T.ObjectUtil.isNotNull(job)) { + pcap.setJobId(job.getId()); + + RunnerEntity runner = runnerService.getById(job.getRunnerId()); + pcap.setRunner(runner); + + PackageEntity pkg = packageService.getById(job.getPackageId()); + pcap.setPkg(pkg); + + PlaybookEntity playbook = playbookService.getById(job.getPlaybookId()); + pcap.setPlaybook(playbook); + + if (T.ObjectUtil.isNotNull(playbook)) { + ApplicationEntity application = applicationService.getById(playbook.getAppId()); + pcap.setApplication(application); + } + } + return pcap; + } + + @Override + public Page queryList(Map params) { + Page page = T.PageUtil.getPage(params); + List pcapList = this.getBaseMapper().queryList(page, params); + page.setRecords(pcapList); + return page; + } @Override - public void savePcap(String jobId, InputStream inputStream) { - // TODO - + public PcapEntity savePcap(String jobId, Resource fileResource) { + JobEntity job = jobService.getById(jobId); + return this.savePcap(fileResource, job.getTags(), job.getWorkbookId(), job.getWorkspaceId(), job.getCreateUserId()); } -} + @Override + public PcapEntity savePcap(Resource fileResource, String... params) { + String tags = T.ArrayUtil.get(params, 0); + String workbookId = T.ArrayUtil.get(params, 1); + String workspaceId = T.ArrayUtil.get(params, 2); + String createUserId = T.StrUtil.emptyToDefault(T.ArrayUtil.get(params, 3), StpUtil.getLoginIdAsString()); + + PcapEntity entity = new PcapEntity(); + try { + entity.setName(fileResource.getFilename()); + entity.setTags(T.StrUtil.emptyToDefault(tags, "")); + + byte[] bytes = fileResource.getInputStream().readAllBytes(); + entity.setSize((long) bytes.length); + + entity.setStatus(RunnerConstant.PcapStatus.UPLOADED.getValue()); + entity.setCreateTimestamp(System.currentTimeMillis()); + entity.setCreateUserId(createUserId); + entity.setWorkspaceId(workspaceId); + + // path + File destination = T.FileUtil.file(T.WebPathUtil.getRootPath(), workspaceId, fileResource.getFilename()); + FileUtils.copyInputStreamToFile(fileResource.getInputStream(), destination); + entity.setPath(destination.getPath()); + + // md5 + String md5Hex = T.DigestUtil.md5Hex(destination); + entity.setMd5(md5Hex); + + // 根据文件 md5值 判断是否已上存在，存在则响应当前实体 + PcapEntity findPcapByMd5 = this.getOne(new LambdaQueryWrapper().eq(PcapEntity::getMd5, md5Hex)); + if (T.ObjectUtil.isNotNull(findPcapByMd5)) { + // 删除本次记录的文件 + T.FileUtil.del(destination); + return findPcapByMd5; + } + + // save + this.save(entity); + + // workbook resource + workbookResourceService.saveResource(workbookId, entity.getId(), WorkbookConstant.ResourceType.PCAP.getValue()); + + // parser + PcapParserThread pcapParserThread = new PcapParserThread(); + pcapParserThread.setPcapEntity(entity); + T.ThreadUtil.execAsync(pcapParserThread); + } catch (IOException e) { + log.error(e, "[savePcap] [error] [workspaceId: {}]", workspaceId); + } + return entity; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deletePcap(String pcapId) { + PcapEntity pcap = this.getById(pcapId); + // remove file + T.FileUtil.del(pcap.getPath()); + + // remove + this.removeById(pcapId); + + // update job pcap_id + jobService.update(new LambdaUpdateWrapper() + .set(JobEntity::getPcapId, "") + .eq(JobEntity::getPcapId, pcapId) + ); + } + +} \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/util/PcapParserThread.java b/src/main/java/net/geedge/asw/module/runner/util/PcapParserThread.java new file mode 100644 index 0000000..8dcbcba --- /dev/null +++ b/src/main/java/net/geedge/asw/module/runner/util/PcapParserThread.java @@ -0,0 +1,80 @@ +package net.geedge.asw.module.runner.util; + +import cn.hutool.log.Log; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import lombok.Data; +import net.geedge.asw.common.config.SpringContextUtils; +import net.geedge.asw.common.util.T; +import net.geedge.asw.module.runner.entity.PcapEntity; +import net.geedge.asw.module.runner.service.IPcapService; + +import java.io.File; + +@Data +public class PcapParserThread implements Runnable { + + private Log log = Log.get(); + + private PcapEntity pcapEntity; + private IPcapService pcapService; + + private void init() { + pcapService = SpringContextUtils.getBean(IPcapService.class); + // analyzing + this.updatePcapStatus(RunnerConstant.PcapStatus.ANALYZING.getValue()); + } + + @Override + public void run() { + Thread.currentThread().setName("pcap-parser-thread-" + pcapEntity.getId()); + log.info("job pcap parser start"); + if (log.isDebugEnabled()) { + log.debug("pcapInfo: {}", T.JSONUtil.toJsonStr(pcapEntity)); + } + try { + log.info("job pcap parser run start"); + // init + this.init(); + // parser + this.parser(); + log.info("job pcap parser run end"); + } catch (Exception e) { + log.error(e, "job pcap parser error, pcap: {}", pcapEntity.getId()); + } finally { + // completed + this.updatePcapStatus(RunnerConstant.PcapStatus.COMPLETED.getValue()); + log.info("job pcap parser end"); + } + } + + /** + * parser + */ + private void parser() { + String id = pcapEntity.getId(); + String path = pcapEntity.getPath(); + SignatureExtract signatureExtract = new SignatureExtract(id, path); + // signature + String signature = signatureExtract.signature(); + // 保存结果，和 pcap 文件同目录，文件名：pcap_id_signature.json + String parentPath = T.FileUtil.getParent(path, 1); + File signatureFile = T.FileUtil.file(parentPath, id + "_signature.json"); + T.FileUtil.del(signatureFile); + T.FileUtil.writeUtf8String(signature, signatureFile); + + // TODO + } + + + /** + * update pcap status + * + * @param status + */ + private void updatePcapStatus(String status) { + pcapService.update(new LambdaUpdateWrapper() + .set(PcapEntity::getStatus, status) + .eq(PcapEntity::getId, pcapEntity.getId()) + ); + } +} \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/util/SignatureExtract.java b/src/main/java/net/geedge/asw/module/runner/util/SignatureExtract.java new file mode 100644 index 0000000..a38b297 --- /dev/null +++ b/src/main/java/net/geedge/asw/module/runner/util/SignatureExtract.java @@ -0,0 +1,179 @@ +package net.geedge.asw.module.runner.util; + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONConfig; +import cn.hutool.log.Log; +import lombok.Data; +import lombok.NoArgsConstructor; +import net.geedge.asw.common.config.SpringContextUtils; +import net.geedge.asw.common.util.ASWException; +import net.geedge.asw.common.util.T; +import org.apache.commons.lang3.time.StopWatch; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +public class SignatureExtract { + + private static final Log log = Log.get(); + + private String id; + private String path; + + private String tsharkPath = "/usr/bin/tshark"; + + public SignatureExtract(String id, String path) { + this.id = id; + this.path = path; + } + + /** + * signature + * + * @return + */ + public String signature() { + log.info("[signature] [begin] [id: {}] [path: {}]", id, path); + StopWatch sw = new StopWatch(); + sw.start(); + try { + this.tsharkPath = SpringContextUtils.getProperty("tshark.path", "/usr/bin/tshark"); + log.info("[signature] [tshark path: {}]", this.tsharkPath); + + SignatureUtil signatureObject = new SignatureUtil(this.tsharkPath); + signatureObject.getStreamSignatureFromTshrak(path); + + List> allFrameSignatureDictList = signatureObject.getOutputDictList(); + + // Get basic information of TCP data streams + List> tcpStreamBasicInfoList = SignatureUtil.getTCPStreamBaseInfo(allFrameSignatureDictList); + List> tcpStreamAllInfoList = T.ListUtil.list(false, tcpStreamBasicInfoList); + + // Get other information of TCP data streams + // Processing data stream by stream + for (int i = 0; i < tcpStreamAllInfoList.size(); i++) { + String streamID = T.MapUtil.getStr(tcpStreamAllInfoList.get(i), "StreamID"); + // Get all the Frame IDs of the data stream + List> tcpFrameSignatureList = signatureObject.getOneTcpFrameSignatureList(streamID); + + // Merge signature information from all Frame IDs + // TCP data flow analysis + this.tcpDataFlowAnalysis(signatureObject, tcpStreamAllInfoList.get(i), tcpFrameSignatureList); + // General data flow analysis (common, ip, dns, http, ssl) + this.generalDataFlowAnalysis(signatureObject, tcpStreamAllInfoList.get(i), tcpFrameSignatureList); + } + + // Get basic information of UDP data streams + List> udpStreamBaseInfo = SignatureUtil.getUDPStreamBaseInfo(allFrameSignatureDictList); + List> udpStreamAllInfoList = T.ListUtil.list(false, udpStreamBaseInfo); + // Get other information of UDP data streams + // Processing data stream by stream + for (int i = 0; i < udpStreamAllInfoList.size(); i++) { + String streamID = T.MapUtil.getStr(udpStreamAllInfoList.get(i), "StreamID"); + // Get all the Frame IDs of the data stream + List> udpFrameSignatureList = signatureObject.getOneUdpFrameSignatureList(streamID); + // Merge signature information from all Frame IDs + // UDP data flow analysis + this.udpDataFlowAnalysis(signatureObject, udpStreamAllInfoList.get(i), udpFrameSignatureList); + // General data flow analysis (common, ip, dns, http, ssl) + this.generalDataFlowAnalysis(signatureObject, udpStreamAllInfoList.get(i), udpFrameSignatureList); + } + + // result + List resultOutputDict = T.ListUtil.list(true); + resultOutputDict.addAll(tcpStreamAllInfoList); + resultOutputDict.addAll(udpStreamAllInfoList); + + JSONConfig jsonConfig = new JSONConfig(); + jsonConfig.setKeyComparator(Comparator.comparing(String::toString)); + JSONArray jsonArray = new JSONArray(resultOutputDict, jsonConfig); + return jsonArray.toJSONString(0); + } catch (Exception e) { + log.error(e, "[signature] [error] [id: {}] [path: {}]", id, path); + throw new ASWException("pcap file parse error. pcap id: " + id); + } finally { + sw.stop(); + log.info("[signature] [finshed] [id: {}] [Run Time: {}]", id, sw.toString()); + } + } + + /** + * data field + * + * @param signatureObject + * @param streamDict + * @param frameSignatureList + */ + private void generalDataFlowAnalysis(SignatureUtil signatureObject, Map streamDict, List> frameSignatureList) { + // common + streamDict.put("common.server_fqdn", signatureObject.ssl_extensions_server_name(frameSignatureList)); + streamDict.put("common.app_id", new String[]{"unknow"}); + if (T.MapUtil.getStr(frameSignatureList.get(0), "ip.proto").equals("6")) { + streamDict.put("srcport", signatureObject.tcp_srcport(frameSignatureList)); + streamDict.put("dstport", signatureObject.tcp_dstport(frameSignatureList)); + } else { + streamDict.put("srcport", signatureObject.udp_srcport(frameSignatureList)); + streamDict.put("dstport", signatureObject.udp_dstport(frameSignatureList)); + } + // ip + streamDict.put("ip.src", signatureObject.ip_src(frameSignatureList)); + streamDict.put("ip.dst", signatureObject.ip_dst(frameSignatureList)); + streamDict.put("ip.proto", signatureObject.ip_proto(frameSignatureList)); + streamDict.put("heartbeat_flag", signatureObject.heartbeat_flag(frameSignatureList)); + // dns + streamDict.put("dns.qry.name", signatureObject.dns_qry_name(frameSignatureList)); + // http + streamDict.put("http.request.full_uri", signatureObject.http_request_full_uri(frameSignatureList)); + streamDict.put("http.request.header", signatureObject.http_request_header(frameSignatureList)); + streamDict.put("http.response.header", signatureObject.http_response_header(frameSignatureList)); + // ssl + streamDict.put("ssl.handshake.certificate.algorithm_identifier", signatureObject.ssl_algorithm_identifier(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.serial_number", signatureObject.ssl_serial_number(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.issuer_common_name", signatureObject.ssl_issuer_common_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.issuer_organization_name", signatureObject.ssl_issuer_organization_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.issuer_country_name", signatureObject.ssl_issuer_country_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.subject_common_name", signatureObject.ssl_subject_common_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.subject_organization_name", signatureObject.ssl_subject_organization_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.subject_country_name", signatureObject.ssl_subject_country_name(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.not_valid_before", signatureObject.ssl_not_valid_before(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.not_valid_after", signatureObject.ssl_not_valid_after(frameSignatureList)); + streamDict.put("ssl.handshake.certificate.algorithm_id", signatureObject.ssl_algorithm_id(frameSignatureList)); + streamDict.put("ssl.analysis.ja3", signatureObject.ssl_ja3(frameSignatureList)); + streamDict.put("ssl.analysis.sni_absent", signatureObject.ssl_sni_absent(frameSignatureList)); + streamDict.put("ssl.analysis.ech_enabled", signatureObject.ssl_ech_enabled(frameSignatureList)); + streamDict.put("ssl.analysis.esni_enabled", signatureObject.ssl_analysis_esni_enabled(frameSignatureList)); + } + + /** + * udp + * + * @param signatureObject + * @param streamDict + * @param frameSignatureList + */ + private void udpDataFlowAnalysis(SignatureUtil signatureObject, Map streamDict, List> frameSignatureList) { + streamDict.put("udp.payload.c2s_first_data", signatureObject.udp_c2s_first_data(frameSignatureList)); + streamDict.put("udp.payload.s2c_first_data", signatureObject.udp_s2c_first_data(frameSignatureList)); + streamDict.put("udp.payload.c2s_first_data_len", signatureObject.udp_c2s_first_data_len(frameSignatureList)); + streamDict.put("udp.payload.s2c_first_data_len", signatureObject.udp_s2c_first_data_len(frameSignatureList)); + streamDict.put("udp.payload", signatureObject.udp_get_payload(frameSignatureList)); + } + + /** + * tcp + * + * @param signatureObject + * @param streamDict + * @param frameSignatureList + */ + private void tcpDataFlowAnalysis(SignatureUtil signatureObject, Map streamDict, List> frameSignatureList) { + streamDict.put("tcp.payload.c2s_first_data", signatureObject.tcp_c2s_first_data(frameSignatureList)); + streamDict.put("tcp.payload.s2c_first_data", signatureObject.tcp_s2c_first_data(frameSignatureList)); + streamDict.put("tcp.payload.c2s_first_data_len", signatureObject.tcp_c2s_first_data_len(frameSignatureList)); + streamDict.put("tcp.payload.s2c_first_data_len", signatureObject.tcp_s2c_first_data_len(frameSignatureList)); + streamDict.put("tcp.payload", signatureObject.tcp_get_payload(frameSignatureList)); + } +} \ No newline at end of file diff --git a/src/main/java/net/geedge/asw/module/runner/util/SignatureUtil.java b/src/main/java/net/geedge/asw/module/runner/util/SignatureUtil.java new file mode 100644 index 0000000..fd0de03 --- /dev/null +++ b/src/main/java/net/geedge/asw/module/runner/util/SignatureUtil.java @@ -0,0 +1,1075 @@ +package net.geedge.asw.module.runner.util; + +import cn.hutool.log.Log; +import lombok.Data; +import net.geedge.asw.common.util.T; + +import javax.security.auth.x500.X500Principal; +import java.io.ByteArrayInputStream; +import java.nio.charset.Charset; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * tshark 解析 pcap 文件 + */ +@Data +public class SignatureUtil { + + private static final Log log = Log.get(); + + private List> outputDictList = T.ListUtil.list(true); + + /** + * algorithm_identifier and algorithm_id mapping relationship dictionary + */ + private static Map ALGORITHM_IDENTIFIER_DICT; + + private String tsharkPath; + static { + ALGORITHM_IDENTIFIER_DICT = new HashMap<>(); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.2", "md4WithRSA"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.3", "md5WithRSA"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.4", "md4WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.6", "desECB"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.11", "rsaSignature"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.14", "mdc2WithRSASignature"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.15", "shaWithRSASignature"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.16", "dhWithCommonModulus"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.17", "desEDE"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.18", "sha"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.19", "mdc-2"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.20", "dsaCommon"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.21", "dsaCommonWithSHA"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.22", "rsaKeyTransport"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.23", "keyed-hash-seal"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.24", "md2WithRSASignature"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.25", "md5WithRSASignature"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.26", "SHA-1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.27", "dsaWithSHA1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.28", "dsaWithCommonSHA1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.14.3.2.29", "sha-1WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10040.4.1", "id-dsa"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10046.2.1", "dhpublicnumber"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.2.1.1.22", "id-keyExchangeAlgorithm"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.2.1", "id-ecPublicKey"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.1.12", "id-ecDH"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.2.13", "id-ecMQV"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.10", "id-RSASSA-PSS"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.8", "id-mgf1"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.1.1", "prime-field"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.2.2", "md2"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.2.4", "md4"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.2.5", "md5"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.1", "rsaEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.2", "md2WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.3", "md4WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.4", "md5WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.5", "sha1WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.6", "rsaOAEPEncryptionSET"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.11", "sha256WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.12", "sha384WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.13", "sha512WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.113549.1.1.14", "sha224WithRSAEncryption"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.4.1", "ecdsa-with-SHA1"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.156.10197.1.501", "SM2-with-SM3"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.4.3.1", "ecdsa-with-SHA224"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.4.3.2", "ecdsa-with-SHA256"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.4.3.3", "ecdsa-with-SHA384"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.4.3.4", "ecdsa-with-SHA512"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.3.1", "id-dsa-with-sha224"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.3.2", "id-dsa-with-sha256"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.3.1.1", "secp192r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.1", "sect163k1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.15", "sect163r2"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.33", "secp224r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.26", "sect233k1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.27", "sect233r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.840.10045.3.1.7", "secp256r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.16", "sect283k1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.17", "sect283r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.34", "secp384r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.36", "sect409k1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.37", "sect409r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.35", "secp521r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.38", "sect571k1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.132.0.39", "sect571r1"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.156.10197.1.301", "sm2"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.2.1", "sha256"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.2.2", "sha384"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.2.3", "sha512"); + ALGORITHM_IDENTIFIER_DICT.put("2.16.840.1.101.3.4.2.4", "sha224"); + ALGORITHM_IDENTIFIER_DICT.put("1.2.156.10197.1.401", "sm3"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.7.4.4", "dilithium2"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.7.1", "p256_dilithium2"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.7.2", "rsa3072_dilithium2"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.7.6.5", "dilithium3"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.7.3", "p384_dilithium3"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.7.8.7", "dilithium5"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.7.4", "p521_dilithium5"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.11.4.4", "dilithium2_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.11.1", "p256_dilithium2_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.11.2", "rsa3072_dilithium2_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.11.6.5", "dilithium3_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.11.3", "p384_dilithium3_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.2.267.11.8.7", "dilithium5_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.2.11.4", "p521_dilithium5_aes"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.3.1", "falcon512"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.3.2", "p256_falcon512"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.3.3", "rsa3072_falcon512"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.3.4", "falcon1024"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.3.5", "p521_falcon1024"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.7", "picnicl1full"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.8", "p256_picnicl1full"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.9", "rsa3072_picnicl1full"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.21", "picnic3l1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.22", "p256_picnic3l1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.6.1.4.1.311.89.2.1.23", "rsa3072_picnic3l1"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.5.1.1.1", "rainbowIclassic"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.5.1.2.1", "p256_rainbowIclassic"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.5.1.3.1", "rsa3072_rainbowIclassic"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.5.3.1.1", "rainbowVclassic"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.5.3.2.1", "p521_rainbowVclassic"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.1.1", "sphincsharaka128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.1.2", "p256_sphincsharaka128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.1.3", "rsa3072_sphincsharaka128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.4.1", "sphincssha256128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.4.2", "p256_sphincssha256128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.4.3", "rsa3072_sphincssha256128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.7.1", "sphincsshake256128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.7.2", "p256_sphincsshake256128frobust"); + ALGORITHM_IDENTIFIER_DICT.put("1.3.9999.6.7.3", "rsa3072_sphincsshake256128frobust"); + } + + public SignatureUtil(String tsharkPath) { + this.tsharkPath = tsharkPath; + } + + /** + * tshark parser pcap + * + * @param pacpFilePath + */ + public void getStreamSignatureFromTshrak(String pacpFilePath) { + List tsharkFilterList = T.ListUtil.list(true); + tsharkFilterList.add("frame.number"); + tsharkFilterList.add("frame.time"); + tsharkFilterList.add("tcp.stream"); + tsharkFilterList.add("udp.stream"); + tsharkFilterList.add("ip.src"); + tsharkFilterList.add("ip.dst"); + tsharkFilterList.add("ip.proto"); + tsharkFilterList.add("data"); + tsharkFilterList.add("udp.srcport"); + tsharkFilterList.add("udp.dstport"); + tsharkFilterList.add("udp.payload"); + tsharkFilterList.add("udp.length"); + tsharkFilterList.add("tcp.srcport"); + tsharkFilterList.add("tcp.dstport"); + tsharkFilterList.add("tcp.payload"); + tsharkFilterList.add("tcp.len"); + tsharkFilterList.add("tcp.flags.syn"); + tsharkFilterList.add("tcp.flags.ack"); + tsharkFilterList.add("tcp.flags.reset"); + tsharkFilterList.add("tcp.flags.fin"); + tsharkFilterList.add("tcp.window_size_value"); + tsharkFilterList.add("tls.record.content_type"); + tsharkFilterList.add("tls.handshake.extensions_server_name"); + tsharkFilterList.add("tls.handshake.session_id"); + tsharkFilterList.add("tls.handshake.type"); + tsharkFilterList.add("tls.handshake.ja3"); + tsharkFilterList.add("tls.handshake.certificate"); + tsharkFilterList.add("tls.esni.encrypted_sni"); + tsharkFilterList.add("x509af.serialNumber"); + tsharkFilterList.add("x509af.algorithm.id"); + tsharkFilterList.add("x509ce.dNSName"); + tsharkFilterList.add("dns.qry.name"); + tsharkFilterList.add("http.request"); + tsharkFilterList.add("http.response"); + tsharkFilterList.add("http.request.full_uri"); + tsharkFilterList.add("http.file_data"); + tsharkFilterList.add("quic"); + + String tsharkFilterStr = ""; + for (String s : tsharkFilterList) { + tsharkFilterStr += " -e " + s; + } + + String cmd = tsharkPath + " -r " + pacpFilePath + " -T fields" + tsharkFilterStr; + if (log.isDebugEnabled()) { + log.debug("cmd: {}", cmd); + } + + List> outputList = this.executeCommand(cmd); + + // All tshark output fields are stored in the dictionary + for (int i = 0; i < outputList.size(); i++) { + Map frameOutputDict = new HashMap<>(); + List strings = outputList.get(i); + for (int j = 0; j < strings.size(); j++) { + frameOutputDict.put(tsharkFilterList.get(j), outputList.get(i).get(j)); + } + outputDictList.add(frameOutputDict); + } + } + + /** + * execute Command + * + * @param cmd + * @return + */ + private List> executeCommand(String cmd) { + Process exec = T.RuntimeUtil.exec(cmd); + List resultLines = T.RuntimeUtil.getResultLines(exec); + List> output = T.ListUtil.list(true); + for (String resultLine : resultLines) { + String[] split = resultLine.split("\t"); + output.add(Arrays.asList(split)); + } + return output; + } + + /** + * tcpFrameSignatureList + * + * @param streamID + * @return + */ + public List> getOneTcpFrameSignatureList(String streamID) { + return outputDictList.stream().filter(map -> T.StrUtil.equals(streamID, T.MapUtil.getStr(map, "tcp.stream"))).collect(Collectors.toList()); + + } + + + // ############################################################################## TCP Stream Info ############################################################################## + public static List> getTCPStreamBaseInfo(List> allFrameSignatureDictList) { + // Extract TCP stream ID list + List> tcpStreamBasicInfoList = T.ListUtil.list(true); + + List tcpStreamIDList = allFrameSignatureDictList.stream().map(entry -> entry.getOrDefault("tcp.stream", "")).filter(s -> T.StrUtil.isNotEmpty(s)).distinct().collect(Collectors.toList()); + + // Extract basic information of TCP data stream + for (int i = 0; i < tcpStreamIDList.size(); i++) { + List> tcpFeatureList = T.ListUtil.list(true); + for (int j = 0; j < allFrameSignatureDictList.size(); j++) { + if (T.StrUtil.equals(allFrameSignatureDictList.get(j).get("tcp.stream"), tcpStreamIDList.get(i))) { + tcpFeatureList.add(allFrameSignatureDictList.get(j)); + } + } + String streamType = "TCP"; + String streamID = tcpStreamIDList.get(i); + String startTag = ""; + // tcp.flags.syn==1 && tcp.flags.ack==0 + // tcp.flags.syn==1 && tcp.flags.ack==1 + // tcp.flags.syn==0 && tcp.flags.ack==1 + if (tcpFeatureList.size() >= 3) { + Map first = tcpFeatureList.get(0); + Map second = tcpFeatureList.get(1); + Map third = tcpFeatureList.get(2); + if (first.get("tcp.flags.syn") == "1" && first.get("tcp.flags.ack") == "0" && second.get("tcp.flags.syn") == "1" && second.get("tcp.flags.ack") == "1" && third.get("tcp.flags.syn") == "0" && third.get("tcp.flags.ack") == "1") { + startTag = "Normal Handshake"; + } else { + startTag = "No Handshake"; + } + } else { + startTag = "frame_num < 3"; + } + + // tcp.flags.fin==0 && tcp.flags.ack==1 + // tcp.flags.fin==1 && tcp.flags.ack==1 + // tcp.flags.fin==0 && tcp.flags.ack==1 + // tcp.flags.fin==1 && tcp.flags.ack==0 + // Four times wave conditions + String endTag = ""; + if (tcpFeatureList.size() >= 4) { + List> ttTcpFeatureList = T.ListUtil.list(false, tcpFeatureList); + List> reversed = ttTcpFeatureList.reversed(); + if (reversed.get(0).get("tcp.flags.reset") == "1") { + endTag = "By Reset"; + } else if (reversed.get(0).get("tcp.flags.fin") == "0" && reversed.get(0).get("tcp.flags.ack") == "1" && reversed.get(1).get("tcp.flags.fin") == "1" && reversed.get(1).get("tcp.flags.ack") == "1" && reversed.get(2).get("tcp.flags.fin") == "0" && reversed.get(2).get("tcp.flags.ack") == "1" && reversed.get(3).get("tcp.flags.fin") == "1" && reversed.get(3).get("tcp.flags.ack") == "0") { + endTag = "Normal Wavehand"; + } else { + endTag = "No Wavehand"; + } + } else { + endTag = "frame_num < 4"; + } + + // Determine the client IP and server IP + String clientIP, clientPort, serverIP, serverPort = ""; + if (T.StrUtil.equals(startTag, "Normal Handshake")) { + Map map = tcpFeatureList.get(0); + clientIP = map.get("ip.src"); + clientPort = map.get("tcp.srcport"); + serverIP = map.get("ip.dst"); + serverPort = map.get("tcp.dstport"); + } else { + Map map = tcpFeatureList.get(0); + if (T.MapUtil.getInt(map, "tcp.srcport") > 5000) { + clientIP = map.get("ip.src"); + clientPort = map.get("tcp.srcport"); + serverIP = map.get("ip.dst"); + serverPort = map.get("tcp.dstport"); + } else { + clientIP = map.get("ip.dst"); + clientPort = map.get("tcp.dstport"); + serverIP = map.get("ip.src"); + serverPort = map.get("tcp.srcport"); + } + } + + // Get the data flow direction stream_irection + List ipList = T.ListUtil.list(true); + for (int j = 0; j < tcpFeatureList.size(); j++) { + String ipSrc = tcpFeatureList.get(j).get("ip.src"); + if (!ipList.contains(ipSrc)) { + ipList.add(ipSrc); + } + } + String streamIrection = ""; + if (ipList.size() > 1) { + streamIrection = "double"; + } else if ("127.0.0.1".equals(ipList.get(0))) { + streamIrection = "local"; + } else { + if (clientIP.equals(ipList.get(0))) { + streamIrection = "c2s"; + } else { + streamIrection = "s2c"; + } + } + + // Get data flow in each direction: s2c_frames_num, s2c_byte, c2s_frames_num, c2s_byte, total_frames_num, total_byte + int s2CFramesNum = 0; + int s2CByte = 0; + int c2SFramesNum = 0; + int c2SByte = 0; + int totalFramesNum = tcpFeatureList.size(); + int totalByte = 0; + for (int j = 0; j < tcpFeatureList.size(); j++) { + int tcpLen = 0; + if (T.StrUtil.isEmpty(tcpFeatureList.get(j).get("tcp.len"))) { + tcpLen = tcpFeatureList.get(j).get("tcp.payload").length(); + } else { + tcpLen = T.MapUtil.getInt(tcpFeatureList.get(j), "tcp.len"); + } + totalByte += tcpLen; + if (clientIP.equals(tcpFeatureList.get(j).get("ip.src"))) { + c2SFramesNum += 1; + c2SByte += tcpLen; + } else { + s2CFramesNum += 1; + s2CByte += tcpLen; + } + } + + // Get the start time and duration of a data stream: relative_start_dt, duration_dt + Object[] retGetDataFlowTime = getDataFlowTime(tcpFeatureList); + String relativeStartDt = (String) retGetDataFlowTime[0]; + String relativeEndDt = (String) retGetDataFlowTime[1]; + Double durationDt = (Double) retGetDataFlowTime[2]; + + Map tcpStreamBaseDict = new HashMap<>(); + tcpStreamBaseDict.put("StreamType", streamType); + tcpStreamBaseDict.put("StreamID", streamID); + tcpStreamBaseDict.put("StartTag", startTag); + tcpStreamBaseDict.put("EndTag", endTag); + tcpStreamBaseDict.put("ClientIP", clientIP); + tcpStreamBaseDict.put("ClientPort", clientPort); + tcpStreamBaseDict.put("ServerIP", serverIP); + tcpStreamBaseDict.put("ServerPort", serverPort); + tcpStreamBaseDict.put("StreamIrection", streamIrection); + tcpStreamBaseDict.put("S2C Num", s2CFramesNum); + tcpStreamBaseDict.put("S2C Byte", s2CByte); + tcpStreamBaseDict.put("C2S Num", c2SFramesNum); + tcpStreamBaseDict.put("C2S Byte", c2SByte); + tcpStreamBaseDict.put("TotalNum", totalFramesNum); + tcpStreamBaseDict.put("TotalByte", totalByte); + tcpStreamBaseDict.put("Start", relativeStartDt); + tcpStreamBaseDict.put("End", relativeEndDt); + tcpStreamBaseDict.put("Duration", durationDt); + tcpStreamBasicInfoList.add(tcpStreamBaseDict); + } + return tcpStreamBasicInfoList; + } + + /** + * data flow time + * + * @param featureList + * @return + */ + private static Object[] getDataFlowTime(List> featureList) { + // Get the start time of the data stream + String frameTimeFirst = T.StrUtil.trim(featureList.get(0).get("frame.time")); + frameTimeFirst = frameTimeFirst.replace("\"", ""); + String[] relativeStartDtList = frameTimeFirst.split("\\s+"); + + String year = relativeStartDtList[2]; + String month = monthToNumber(relativeStartDtList[0]); + String day = relativeStartDtList[1].split(",")[0]; + + String element = relativeStartDtList[3]; + String timeStr = element.substring(0, element.length() - 3); + String relativeStartDt = year + '-' + month + '-' + day + ' ' + timeStr; + + // Get the end time of the data stream + String fullTimeLast = T.StrUtil.trim(featureList.getLast().get("frame.time")); + fullTimeLast = fullTimeLast.replace("\"", ""); + String[] relativeEndDtList = fullTimeLast.split("\\s+"); + year = relativeEndDtList[2]; + month = monthToNumber(relativeEndDtList[0]); + day = relativeEndDtList[1].split(",")[0]; + + String element1 = relativeEndDtList[3]; + timeStr = element1.substring(0, element1.length() - 3); + String relativeEndDt = year + '-' + month + '-' + day + ' ' + timeStr; + + // Get the duration of a data stream + long timeDifference = T.DateUtil.parse(relativeEndDt).getTime() - T.DateUtil.parse(relativeStartDt).getTime(); + Double durationDt = timeDifference / 1000.0; // Convert milliseconds to seconds + return new Object[]{relativeStartDt, relativeEndDt, durationDt}; + } + + /** + * month to number + * + * @param abbreviation + * @return + */ + private static String monthToNumber(String abbreviation) { + Map months = new HashMap<>(); + months.put("JAN", 1); + months.put("FEB", 2); + months.put("MAR", 3); + months.put("APR", 4); + months.put("MAY", 5); + months.put("JUN", 6); + months.put("JUL", 7); + months.put("AUG", 8); + months.put("SEP", 9); + months.put("OCT", 10); + months.put("NOV", 11); + months.put("DEC", 12); + return months.get(abbreviation.toUpperCase()) + ""; + } + + /** + * hexStr to http headers + * + * @param string + * @return + */ + private static Map hexToHttpHeaders(String string) { + String httpHeadersStr = T.HexUtil.decodeHexStr(string, Charset.forName("utf-8")); + String[] lines = httpHeadersStr.split("\r\n"); + Map headers = new HashMap<>(); + for (String line : lines) { + if (line.contains(": ")) { + String[] split = line.split(": ", 2); + headers.put(split[0], split[1]); + } + } + return headers; + } + + + // ############################################################################## UDP Stream Info ############################################################################## + public static List> getUDPStreamBaseInfo(List> allFrameSignatureDictList) { + List> udpStreamBasicInfoList = T.ListUtil.list(true); + // Extract UDP stream ID list + List udpStreamIDList = allFrameSignatureDictList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "udp.stream"))).map(map -> T.MapUtil.getStr(map, "udp.stream")).distinct().collect(Collectors.toList()); + + // Extract basic information of udp data stream + for (String streamID : udpStreamIDList) { + List> udpFeatureList = allFrameSignatureDictList.stream().filter(map -> T.StrUtil.equals(streamID, T.MapUtil.getStr(map, "udp.stream"))).collect(Collectors.toList()); + + String streamType = "UDP"; + String startTag = "Normal"; + String endTag = "Normal"; + + String clientIP, clientPort, serverIP, serverPort = ""; + // Determine the client IP and server IP. The one with the larger port in UDP is defined as the client. + if (T.MapUtil.getInt(udpFeatureList.get(0), "udp.srcport", 0) > T.MapUtil.getInt(udpFeatureList.get(0), "udp.dstport", 0)) { + clientIP = udpFeatureList.get(0).get("ip.src"); + clientPort = udpFeatureList.get(0).get("udp.srcport"); + serverIP = udpFeatureList.get(0).get("ip.dst"); + serverPort = udpFeatureList.get(0).get("udp.dstport"); + } else { + clientIP = udpFeatureList.get(0).get("ip.dst"); + clientPort = udpFeatureList.get(0).get("udp.dstport"); + serverIP = udpFeatureList.get(0).get("ip.src"); + serverPort = udpFeatureList.get(0).get("udp.srcport"); + } + + // Determine the direction of data flow + String streamIrection = ""; + List ip_list = udpFeatureList.stream().map(map -> T.MapUtil.getStr(map, "ip.src")).distinct().collect(Collectors.toList()); + if (T.CollUtil.size(ip_list) > 1) { + streamIrection = "double"; + } else if (T.StrUtil.equals("127.0.0.1", ip_list.get(0))) { + streamIrection = "local"; + } else { + if (T.StrUtil.equals(ip_list.get(0), clientIP)) { + streamIrection = "c2s"; + } else { + streamIrection = "s2c"; + } + } + + // Get data flow in each direction: s2c_frames_num, s2c_byte, c2s_frames_num, c2s_byte, total_frames_num, total_byte + int s2CFramesNum = 0; + int s2CByte = 0; + int c2SFramesNum = 0; + int c2SByte = 0; + int totalFramesNum = udpFeatureList.size(); + int totalByte = 0; + for (Map map : udpFeatureList) { + totalByte += T.MapUtil.getInt(map, "udp.length", 0); + if (T.StrUtil.equals(clientIP, T.MapUtil.getStr(map, "ip.src"))) { + c2SFramesNum += 1; + c2SByte += T.MapUtil.getInt(map, "udp.length", 0); + } else { + s2CFramesNum += 1; + s2CByte += T.MapUtil.getInt(map, "udp.length", 0); + } + } + + // Get the start time and duration of a data stream relative_start_dt, duration_dt + Object[] retGetDataFlowTime = getDataFlowTime(udpFeatureList); + String relativeStartDt = (String) retGetDataFlowTime[0]; + String relativeEndDt = (String) retGetDataFlowTime[1]; + Double durationDt = (Double) retGetDataFlowTime[2]; + + Map udp_stream_base_dict = new HashMap<>(); + udp_stream_base_dict.put("StreamType", streamType); + udp_stream_base_dict.put("StreamID", streamID); + udp_stream_base_dict.put("StartTag", startTag); + udp_stream_base_dict.put("EndTag", endTag); + udp_stream_base_dict.put("ClientIP", clientIP); + udp_stream_base_dict.put("ClientPort", clientPort); + udp_stream_base_dict.put("ServerIP", serverIP); + udp_stream_base_dict.put("ServerPort", serverPort); + udp_stream_base_dict.put("StreamIrection", streamIrection); + udp_stream_base_dict.put("S2C Num", s2CFramesNum); + udp_stream_base_dict.put("S2C Byte", s2CByte); + udp_stream_base_dict.put("C2S Num", c2SFramesNum); + udp_stream_base_dict.put("C2S Byte", c2SByte); + udp_stream_base_dict.put("TotalNum", totalFramesNum); + udp_stream_base_dict.put("TotalByte", totalByte); + udp_stream_base_dict.put("Start", relativeStartDt); + udp_stream_base_dict.put("End", relativeEndDt); + udp_stream_base_dict.put("Duration", durationDt); + udpStreamBasicInfoList.add(udp_stream_base_dict); + } + return udpStreamBasicInfoList; + } + + public List> getOneUdpFrameSignatureList(String streamID) { + List> tcpFrameSignatureList = T.ListUtil.list(true); + for (Map map : outputDictList) { + if (T.StrUtil.equals(map.get("udp.stream"), streamID)) { + tcpFrameSignatureList.add(map); + } + } + return tcpFrameSignatureList; + } + + private String[] getUdpClinetAndServerIP(List> frameSignatureList) { + String clientIp, clientPort, serverIp, serverPort = ""; + // The client port is larger than the server port + Map first = frameSignatureList.getFirst(); + if (T.MapUtil.getInt(first, "udp.srcport", 0) >= T.MapUtil.getInt(first, "udp.dstport", 0)) { + clientIp = T.MapUtil.getStr(first, "ip.src"); + clientPort = T.MapUtil.getStr(first, "udp.srcport"); + serverIp = T.MapUtil.getStr(first, "ip.dst"); + serverPort = T.MapUtil.getStr(first, "udp.dstport"); + } else { + clientIp = T.MapUtil.getStr(first, "ip.dst"); + clientPort = T.MapUtil.getStr(first, "udp.dstport"); + serverIp = T.MapUtil.getStr(first, "ip.src"); + serverPort = T.MapUtil.getStr(first, "udp.srcport"); + } + return new String[]{clientIp, clientPort, serverIp, serverPort}; + } + + + // ############################################################################## udp ############################################################################## + public String[] udp_c2s_first_data(List> frameSignatureList) { + String[] retList = getUdpClinetAndServerIP(frameSignatureList); + String clientIp = retList[0]; + for (Map map : frameSignatureList) { + if (T.StrUtil.equals(clientIp, T.MapUtil.getStr(map, "ip.src"))) { + String str = T.MapUtil.getStr(map, "udp.payload"); + return new String[]{str}; + } + } + return new String[]{}; + } + + public Object udp_c2s_first_data_len(List> frameSignatureList) { + if (udp_c2s_first_data(frameSignatureList).length > 0) { + return new String[]{T.StrUtil.length(udp_c2s_first_data(frameSignatureList)[0]) + ""}; + } + return new String[]{"0"}; + } + + public String[] udp_s2c_first_data(List> frameSignatureList) { + String[] retList = getUdpClinetAndServerIP(frameSignatureList); + String serverIp = retList[2]; + for (Map map : frameSignatureList) { + if (T.StrUtil.equals(serverIp, T.MapUtil.getStr(map, "ip.src"))) { + String str = T.MapUtil.getStr(map, "udp.payload"); + return new String[]{str}; + } + } + return new String[]{}; + } + + public Object udp_s2c_first_data_len(List> frameSignatureList) { + if (udp_s2c_first_data(frameSignatureList).length > 0) { + return new String[]{T.StrUtil.length(udp_s2c_first_data(frameSignatureList)[0]) + ""}; + } + return new String[]{"0"}; + } + + public List udp_get_payload(List> frameSignatureList) { + return frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "udp.payload"))).map(map -> T.MapUtil.getStr(map, "udp.payload")).collect(Collectors.toList()); + } + + public List udp_srcport(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "udp.srcport")).distinct().collect(Collectors.toList()); + } + + public List udp_dstport(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "udp.dstport")).distinct().collect(Collectors.toList()); + } + + + // ############################################################################## tcp ############################################################################## + public String[] tcp_c2s_first_data(List> frameSignatureList) { + if (T.CollUtil.size(frameSignatureList) > 3) { + if ("create_with_syn".equals(this.tcp_create_with_syn(frameSignatureList))) { + return new String[]{frameSignatureList.get(3).get("tcp.payload")}; + } + return new String[]{""}; + } + return new String[]{""}; + } + + public String[] tcp_c2s_first_data_len(List> frameSignatureList) { + return new String[]{T.StrUtil.length(tcp_c2s_first_data(frameSignatureList)[0]) + ""}; + } + + public String[] tcp_s2c_first_data(List> frameSignatureList) { + if (T.CollUtil.size(frameSignatureList) > 4) { + if ("create_with_syn".equals(this.tcp_create_with_syn(frameSignatureList))) { + return new String[]{frameSignatureList.get(4).get("tcp.payload")}; + } + return new String[]{""}; + } + return new String[]{""}; + } + + public String[] tcp_s2c_first_data_len(List> frameSignatureList) { + return new String[]{T.StrUtil.length(tcp_s2c_first_data(frameSignatureList)[0]) + ""}; + } + + public String tcp_create_with_syn(List> frameSignatureList) { + if (T.CollUtil.size(frameSignatureList) >= 3) { + if (frameSignatureList.get(0).get("tcp.flags.syn") == "1" && frameSignatureList.get(0).get("tcp.flags.ack") == "0" && frameSignatureList.get(1).get("tcp.flags.syn") == "1" && frameSignatureList.get(1).get("tcp.flags.ack") == "1" && frameSignatureList.get(2).get("tcp.flags.syn") == "0" && frameSignatureList.get(2).get("tcp.flags.ack") == "1") { + return "create_with_syn"; + } else { + return "NO create_with_syn"; + } + } + return "TCP NO Create"; + } + + public List tcp_get_payload(List> frameSignatureList) { + List retList = T.ListUtil.list(true); + for (int i = 0; i < frameSignatureList.size(); i++) { + Map map = frameSignatureList.get(i); + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tcp.payload"))) { + retList.add(T.MapUtil.getStr(map, "tcp.payload")); + } + } + return retList; + } + + public List tcp_srcport(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "tcp.srcport")).distinct().collect(Collectors.toList()); + } + + public List tcp_dstport(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "tcp.dstport")).distinct().collect(Collectors.toList()); + } + + + // ############################################################################## ip ############################################################################## + public List ip_src(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "ip.src")).distinct().collect(Collectors.toList()); + } + + public List ip_dst(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "ip.dst")).distinct().collect(Collectors.toList()); + } + + public List ip_proto(List> frameSignatureList) { + List retList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + String proto = T.MapUtil.getStr(map, "ip.proto"); + if (!retList.contains(proto)) { + if ("6".equals(proto)) { + retList.add("TCP"); + break; + } else if ("17".equals(proto)) { + retList.add("UDP"); + break; + } + } + } + return retList; + } + + public List heartbeat_flag(List> frameSignatureList) { + return frameSignatureList.stream().filter(map -> "24".equals(T.MapUtil.getStr(map, "tls.record.content_type"))).map(map -> T.MapUtil.getStr(map, "frame.number")).collect(Collectors.toList()); + } + + + // ############################################################################## dns ############################################################################## + + public List dns_qry_name(List> frameSignatureList) { + return frameSignatureList.stream().map(map -> T.MapUtil.getStr(map, "dns.qry.name")).collect(Collectors.toList()); + } + + + // ############################################################################## http ############################################################################## + public List http_request_full_uri(List> frameSignatureList) { + return frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "http.request.full_uri"))).map(map -> T.MapUtil.getStr(map, "http.request.full_uri")).collect(Collectors.toList()); + } + + public List> http_request_header(List> frameSignatureList) { + List> retList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if ("True".equals(T.MapUtil.getStr(map, "http.request"))) { + String tcpPayload = T.MapUtil.getStr(map, "tcp.payload"); + String httpFileData = T.MapUtil.getStr(map, "http.file_data"); + String modifiedString = tcpPayload.replace(httpFileData, ""); + Map headers = hexToHttpHeaders(modifiedString); + retList.add(headers); + } + } + return retList; + } + + public List> http_response_header(List> frameSignatureList) { + List> retList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if ("True".equals(T.MapUtil.getStr(map, "http.response"))) { + String tcpPayload = T.MapUtil.getStr(map, "tcp.payload"); + String httpFileData = T.MapUtil.getStr(map, "http.file_data"); + String modifiedString = tcpPayload.replace(httpFileData, ""); + Map headers = hexToHttpHeaders(modifiedString); + retList.add(headers); + } + } + return retList; + } + + + // ############################################################################## ssl ############################################################################## + public List ssl_algorithm_identifier(List> frameSignatureList) { + List retListTemp = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "x509af.algorithm.id"))) { + retListTemp.addAll(Arrays.asList(T.MapUtil.getStr(map, "x509af.algorithm.id").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String str : retListTemp) { + String algorithm_identifier_str = T.MapUtil.getStr(ALGORITHM_IDENTIFIER_DICT, str, "unknown"); + retList.add(algorithm_identifier_str); + } + return retList.stream().distinct().collect(Collectors.toList()); + } + + public List ssl_serial_number(List> frameSignatureList) { + return frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "x509af.serialNumber"))).map(map -> T.MapUtil.getStr(map, "x509af.serialNumber")).distinct().collect(Collectors.toList()); + } + + public List ssl_extensions_server_name(List> frameSignatureList) { + List retList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.MapUtil.getStr(map, "quic") != "1" && T.MapUtil.getStr(map, "tls.handshake.extensions_server_name") != "") { + retList.add(T.MapUtil.getStr(map, "tls.handshake.extensions_server_name")); + } + } + retList = retList.stream().distinct().collect(Collectors.toList()); + return retList; + } + + public List ssl_issuer_common_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal issuerX500Principal = cert.getIssuerX500Principal(); + String name = issuerX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("CN", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_issuer_organization_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal issuerX500Principal = cert.getIssuerX500Principal(); + String name = issuerX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("O", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_issuer_country_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal issuerX500Principal = cert.getIssuerX500Principal(); + String name = issuerX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("C", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_subject_common_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal subjectX500Principal = cert.getSubjectX500Principal(); + String name = subjectX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("CN", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_subject_organization_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal subjectX500Principal = cert.getSubjectX500Principal(); + String name = subjectX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("O", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_subject_country_name(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + X500Principal subjectX500Principal = cert.getSubjectX500Principal(); + String name = subjectX500Principal.getName(); + for (String str : name.split(",")) { + String[] split = str.split("="); + if (T.StrUtil.equalsIgnoreCase("C", split[0])) { + retList.add(split[1]); + } + } + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_not_valid_before(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.of("UTC")); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + // 获取证书的生效日期 + Instant instant = Instant.ofEpochMilli(cert.getNotBefore().getTime()); + retList.add(formatter.format(instant)); + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_not_valid_after(List> frameSignatureList) { + try { + List retCertificateList = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.certificate"))) { + retCertificateList.addAll(Arrays.asList(T.MapUtil.getStr(map, "tls.handshake.certificate").split(","))); + } + } + List retList = T.ListUtil.list(true); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.of("UTC")); + for (String certStr : retCertificateList) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + X509Certificate cert = (X509Certificate) cf.generateCertificate(new ByteArrayInputStream(T.HexUtil.decodeHex(certStr))); + // 获取证书的生效日期 + Instant instant = Instant.ofEpochMilli(cert.getNotAfter().getTime()); + retList.add(formatter.format(instant)); + } + return retList; + } catch (CertificateException e) { + log.error(e); + } + return null; + } + + public List ssl_algorithm_id(List> frameSignatureList) { + List retListTemp = T.ListUtil.list(true); + for (Map map : frameSignatureList) { + if (T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "x509af.algorithm.id"))) { + retListTemp.addAll(Arrays.asList(T.MapUtil.getStr(map, "x509af.algorithm.id").split(","))); + } + } + return retListTemp.stream().distinct().collect(Collectors.toList()); + } + + public List ssl_ja3(List> frameSignatureList) { + return frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.ja3"))).map(map -> T.MapUtil.getStr(map, "tls.handshake.ja3")).distinct().collect(Collectors.toList()); + } + + public String[] ssl_sni_absent(List> frameSignatureList) { + long count = frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.handshake.extensions_server_name"))).map(map -> T.MapUtil.getStr(map, "tls.handshake.extensions_server_name")).distinct().count(); + if (0 == count) { + return new String[]{"Ture"}; + } + return new String[]{"False"}; + } + + public String[] ssl_ech_enabled(List> frameSignatureList) { + long count = frameSignatureList.stream().filter(map -> T.StrUtil.equals("8", T.MapUtil.getStr(map, "tls.handshake.type"))).map(map -> T.MapUtil.getStr(map, "tls.handshake.type")).count(); + if (0 == count) { + return new String[]{"False"}; + } + return new String[]{"Ture"}; + } + + public String[] ssl_analysis_esni_enabled(List> frameSignatureList) { + long count = frameSignatureList.stream().filter(map -> T.StrUtil.isNotEmpty(T.MapUtil.getStr(map, "tls.esni.encrypted_sni"))).map(map -> T.MapUtil.getStr(map, "tls.esni.encrypted_sni")).distinct().count(); + if (0 == count) { + return new String[]{"False"}; + } + return new String[]{"Ture"}; + } +} \ No newline at end of file diff --git a/src/main/resources/db/mapper/runner/JobMapper.xml b/src/main/resources/db/mapper/runner/JobMapper.xml index 7d5fbc4..381b180 100644 --- a/src/main/resources/db/mapper/runner/JobMapper.xml +++ b/src/main/resources/db/mapper/runner/JobMapper.xml @@ -5,7 +5,6 @@ - diff --git a/src/main/resources/db/mapper/runner/PcapMapper.xml b/src/main/resources/db/mapper/runner/PcapMapper.xml new file mode 100644 index 0000000..023b135 --- /dev/null +++ b/src/main/resources/db/mapper/runner/PcapMapper.xml @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.0.01__INIT_TABLES.sql b/src/main/resources/db/migration/V1.0.01__INIT_TABLES.sql index 809e903..7146b20 100644 --- a/src/main/resources/db/migration/V1.0.01__INIT_TABLES.sql +++ b/src/main/resources/db/migration/V1.0.01__INIT_TABLES.sql @@ -224,8 +224,8 @@ CREATE TABLE `pcap` ( `connections` bigint(20) NOT NULL DEFAULT 0 COMMENT '连接数量', `hosts` bigint(20) NOT NULL DEFAULT 0 COMMENT 'IP数量', `md5` varchar(64) NOT NULL DEFAULT '' COMMENT '摘要值,根据文件md5值判断是否已上存在，存在则响应当前id', - `connection_time_first` bigint(20) NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW()) * 1000) COMMENT '连接开始时间', - `connection_time_last` bigint(20) NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW()) * 1000) COMMENT '连接结束时间', + `connection_time_first` bigint(20) NOT NULL DEFAULT -1 COMMENT '连接开始时间', + `connection_time_last` bigint(20) NOT NULL DEFAULT -1 COMMENT '连接结束时间', `protocols` varchar(64) NOT NULL DEFAULT '' COMMENT '包含的协议,多个逗号分隔', `status` varchar(64) NOT NULL DEFAULT '' COMMENT '状态,可选值 Uploaded，Analyzing，Completed', `create_timestamp` bigint(20) NOT NULL COMMENT '创建时间戳',