feat(aria): Phase 5 — IOC feed refresh from OTX, MalwareBazaar, ThreatFox
CD - Develop / build-and-deploy (push) Failing after 52s
CD - Develop / build-and-deploy (push) Failing after 52s
Adds automated daily IOC ingestion from three external threat feeds:
- AlienVault OTX: subscribed pulses + ATM-keyword searches (requires API key)
- MalwareBazaar (abuse.ch): ATM-tagged malware samples, no key required
- ThreatFox (abuse.ch): ATM malware family IOCs, no key required
Backend:
- IocFeedRun entity + V6 Flyway migration (ioc_feed_run table)
- OtxFeedClient / MalwareBazaarFeedClient / ThreatFoxFeedClient
- IocFeedService: upsert dedup + @Scheduled daily 03:00 refresh
- IocFeedController: GET /status, GET /runs, POST /refresh, POST /refresh/{feed}
- ThreatIoc.IocSource enum extended: OTX, MALWARE_BAZAAR, THREAT_FOX
Frontend:
- IocFeeds.svelte: feed status cards, run history table, manual refresh
- App.svelte: IOC Feeds nav item (📡), version string removed from sidebar
- api.ts: getFeedStatus, getFeedRuns, refreshAllFeeds, refreshFeed
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
package com.hiveops.aria.controller;
|
||||
|
||||
import com.hiveops.aria.entity.IocFeedRun;
|
||||
import com.hiveops.aria.repository.IocFeedRunRepository;
|
||||
import com.hiveops.aria.service.IocFeedService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.security.access.prepost.PreAuthorize;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/aria/feeds")
|
||||
@RequiredArgsConstructor
|
||||
public class IocFeedController {
|
||||
|
||||
private final IocFeedService feedService;
|
||||
private final IocFeedRunRepository feedRunRepository;
|
||||
|
||||
/** Latest run per feed — used by the status cards in the UI */
|
||||
@GetMapping("/status")
|
||||
@PreAuthorize("hasRole('BCOS_ADMIN')")
|
||||
public ResponseEntity<Map<String, Object>> status() {
|
||||
List<String> names = feedService.feedNames();
|
||||
Map<String, Object> result = new java.util.LinkedHashMap<>();
|
||||
for (String name : names) {
|
||||
Optional<IocFeedRun> last = feedRunRepository.findTopByFeedNameOrderByStartedAtDesc(name);
|
||||
result.put(name, last.orElse(null));
|
||||
}
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
|
||||
/** Recent run history — default 50 rows */
|
||||
@GetMapping("/runs")
|
||||
@PreAuthorize("hasRole('BCOS_ADMIN')")
|
||||
public ResponseEntity<List<IocFeedRun>> runs(
|
||||
@RequestParam(defaultValue = "0") int page,
|
||||
@RequestParam(defaultValue = "50") int size) {
|
||||
return ResponseEntity.ok(
|
||||
feedRunRepository.findAllByOrderByStartedAtDesc(PageRequest.of(page, size)).getContent()
|
||||
);
|
||||
}
|
||||
|
||||
/** Trigger all feeds immediately */
|
||||
@PostMapping("/refresh")
|
||||
@PreAuthorize("hasRole('BCOS_ADMIN')")
|
||||
public ResponseEntity<Map<String, String>> refreshAll() {
|
||||
new Thread(() -> feedService.refreshAll(), "ioc-feed-manual-all").start();
|
||||
return ResponseEntity.accepted().body(Map.of("status", "refresh_started"));
|
||||
}
|
||||
|
||||
/** Trigger one specific feed */
|
||||
@PostMapping("/refresh/{feedName}")
|
||||
@PreAuthorize("hasRole('BCOS_ADMIN')")
|
||||
public ResponseEntity<Map<String, String>> refreshOne(@PathVariable String feedName) {
|
||||
String upper = feedName.toUpperCase();
|
||||
if (!feedService.feedNames().contains(upper)) {
|
||||
return ResponseEntity.badRequest().body(Map.of("error", "Unknown feed: " + feedName));
|
||||
}
|
||||
new Thread(() -> feedService.refreshFeed(upper), "ioc-feed-manual-" + upper).start();
|
||||
return ResponseEntity.accepted().body(Map.of("status", "refresh_started", "feed", upper));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.hiveops.aria.entity;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
@Entity
|
||||
@Table(name = "ioc_feed_run")
|
||||
@Getter
|
||||
@Setter
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
public class IocFeedRun {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(name = "feed_name", nullable = false)
|
||||
private String feedName;
|
||||
|
||||
@Column(name = "started_at", nullable = false)
|
||||
@Builder.Default
|
||||
private Instant startedAt = Instant.now();
|
||||
|
||||
@Column(name = "completed_at")
|
||||
private Instant completedAt;
|
||||
|
||||
@Column(nullable = false)
|
||||
@Builder.Default
|
||||
private String status = "RUNNING";
|
||||
|
||||
@Column(name = "iocs_added", nullable = false)
|
||||
@Builder.Default
|
||||
private int iocsAdded = 0;
|
||||
|
||||
@Column(name = "iocs_updated", nullable = false)
|
||||
@Builder.Default
|
||||
private int iocsUpdated = 0;
|
||||
|
||||
@Column(name = "error_message")
|
||||
private String errorMessage;
|
||||
}
|
||||
@@ -56,7 +56,7 @@ public class ThreatIoc {
|
||||
}
|
||||
|
||||
public enum IocSource {
|
||||
FBI_FLASH, FS_ISAC, MANUAL
|
||||
FBI_FLASH, FS_ISAC, MANUAL, OTX, MALWARE_BAZAAR, THREAT_FOX
|
||||
}
|
||||
|
||||
public enum ConfidenceLevel {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package com.hiveops.aria.feed;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.hiveops.aria.entity.ThreatIoc;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Fetches ATM-related malware samples from MalwareBazaar (abuse.ch) — no API key required.
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class MalwareBazaarFeedClient {
|
||||
|
||||
private static final String API_URL = "https://mb-api.abuse.ch/api/v1/";
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
public List<ThreatIoc> fetch() {
|
||||
List<ThreatIoc> result = new ArrayList<>();
|
||||
|
||||
// Query recent samples tagged with ATM-related tags
|
||||
for (String tag : List.of("ATM", "jackpotting", "NCR", "Diebold", "GreenDispenser", "Tyupkin")) {
|
||||
result.addAll(queryByTag(tag));
|
||||
}
|
||||
|
||||
log.info("MalwareBazaar: fetched {} raw indicators", result.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<ThreatIoc> queryByTag(String tag) {
|
||||
List<ThreatIoc> out = new ArrayList<>();
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
|
||||
MultiValueMap<String, String> body = new LinkedMultiValueMap<>();
|
||||
body.add("query", "get_taginfo");
|
||||
body.add("tag", tag);
|
||||
body.add("limit", "100");
|
||||
|
||||
HttpEntity<MultiValueMap<String, String>> entity = new HttpEntity<>(body, headers);
|
||||
String response = restTemplate.postForObject(API_URL, entity, String.class);
|
||||
|
||||
JsonNode root = MAPPER.readTree(response);
|
||||
if (!"ok".equals(root.path("query_status").asText())) return out;
|
||||
|
||||
for (JsonNode sample : root.path("data")) {
|
||||
String sha256 = sample.path("sha256_hash").asText("").trim();
|
||||
String md5 = sample.path("md5_hash").asText("").trim();
|
||||
String name = sample.path("file_name").asText("").trim();
|
||||
String sigName = sample.path("signature").asText("Unknown");
|
||||
|
||||
if (!md5.isBlank()) {
|
||||
out.add(ThreatIoc.builder()
|
||||
.iocType(ThreatIoc.IocType.MD5)
|
||||
.value(md5)
|
||||
.description("MalwareBazaar tag:" + tag + " sig:" + sigName)
|
||||
.source(ThreatIoc.IocSource.MALWARE_BAZAAR)
|
||||
.sourceRef(sha256.isBlank() ? tag : sha256)
|
||||
.confidence(ThreatIoc.ConfidenceLevel.HIGH)
|
||||
.expiresAt(Instant.now().plus(365, ChronoUnit.DAYS))
|
||||
.build());
|
||||
}
|
||||
if (!name.isBlank()) {
|
||||
out.add(ThreatIoc.builder()
|
||||
.iocType(ThreatIoc.IocType.FILENAME)
|
||||
.value(name)
|
||||
.description("MalwareBazaar tag:" + tag + " sig:" + sigName)
|
||||
.source(ThreatIoc.IocSource.MALWARE_BAZAAR)
|
||||
.sourceRef(sha256.isBlank() ? tag : sha256)
|
||||
.confidence(ThreatIoc.ConfidenceLevel.MEDIUM)
|
||||
.expiresAt(Instant.now().plus(365, ChronoUnit.DAYS))
|
||||
.build());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("MalwareBazaar: error querying tag {}: {}", tag, e.getMessage());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package com.hiveops.aria.feed;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.hiveops.aria.entity.ThreatIoc;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.net.URI;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@Slf4j
|
||||
public class OtxFeedClient {
|
||||
|
||||
private static final String BASE_URL = "https://otx.alienvault.com/api/v1";
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
@Value("${aria.feed.otx.api-key:}")
|
||||
private String apiKey;
|
||||
|
||||
private final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
public boolean isConfigured() {
|
||||
return apiKey != null && !apiKey.isBlank();
|
||||
}
|
||||
|
||||
public List<ThreatIoc> fetch() {
|
||||
if (!isConfigured()) {
|
||||
log.info("OTX: no API key configured, skipping");
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<ThreatIoc> result = new ArrayList<>();
|
||||
String since = Instant.now().minus(8, ChronoUnit.DAYS).toString();
|
||||
|
||||
// Subscribed pulses modified in the last 8 days
|
||||
fetchPulses(BASE_URL + "/pulses/subscribed?modified_since=" + since + "&limit=200", result);
|
||||
|
||||
// ATM-specific keyword searches
|
||||
for (String query : List.of("atm+malware", "jackpotting", "atm+skimming")) {
|
||||
fetchPulses(BASE_URL + "/search/pulses?q=" + query + "&limit=10", result);
|
||||
}
|
||||
|
||||
log.info("OTX: fetched {} raw indicators", result.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
private void fetchPulses(String url, List<ThreatIoc> out) {
|
||||
try {
|
||||
org.springframework.http.HttpHeaders headers = new org.springframework.http.HttpHeaders();
|
||||
headers.set("X-OTX-API-KEY", apiKey);
|
||||
org.springframework.http.HttpEntity<Void> entity = new org.springframework.http.HttpEntity<>(headers);
|
||||
|
||||
org.springframework.http.ResponseEntity<String> response =
|
||||
restTemplate.exchange(URI.create(url),
|
||||
org.springframework.http.HttpMethod.GET, entity, String.class);
|
||||
|
||||
JsonNode root = MAPPER.readTree(response.getBody());
|
||||
JsonNode results = root.path("results");
|
||||
if (results.isMissingNode()) results = root.path("pulse");
|
||||
|
||||
for (JsonNode pulse : results) {
|
||||
JsonNode indicators = pulse.path("indicators");
|
||||
for (JsonNode ind : indicators) {
|
||||
ThreatIoc ioc = mapIndicator(ind, pulse.path("name").asText(""));
|
||||
if (ioc != null) out.add(ioc);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("OTX: error fetching {}: {}", url, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private ThreatIoc mapIndicator(JsonNode ind, String pulseName) {
|
||||
String type = ind.path("type").asText("");
|
||||
String value = ind.path("indicator").asText("").trim();
|
||||
if (value.isBlank()) return null;
|
||||
|
||||
ThreatIoc.IocType iocType = switch (type) {
|
||||
case "FileHash-MD5" -> ThreatIoc.IocType.MD5;
|
||||
case "FilePath" -> ThreatIoc.IocType.FILE_PATH;
|
||||
case "IPv4" -> ThreatIoc.IocType.IP;
|
||||
case "Win-Registry-Key" -> ThreatIoc.IocType.REGISTRY_KEY;
|
||||
default -> null;
|
||||
};
|
||||
if (iocType == null) return null;
|
||||
|
||||
return ThreatIoc.builder()
|
||||
.iocType(iocType)
|
||||
.value(value)
|
||||
.description("OTX pulse: " + pulseName)
|
||||
.source(ThreatIoc.IocSource.OTX)
|
||||
.sourceRef(pulseName)
|
||||
.confidence(ThreatIoc.ConfidenceLevel.MEDIUM)
|
||||
.expiresAt(Instant.now().plus(90, ChronoUnit.DAYS))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package com.hiveops.aria.feed;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.hiveops.aria.entity.ThreatIoc;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Fetches ATM-related IOCs from ThreatFox (abuse.ch) — no API key required.
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class ThreatFoxFeedClient {
|
||||
|
||||
private static final String API_URL = "https://threatfox-api.abuse.ch/api/v1/";
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
private final RestTemplate restTemplate = new RestTemplate();
|
||||
|
||||
public List<ThreatIoc> fetch() {
|
||||
List<ThreatIoc> result = new ArrayList<>();
|
||||
|
||||
// Search by ATM-specific malware families
|
||||
for (String malware : List.of("ATMii", "GreenDispenser", "Tyupkin", "Ploutus", "SUCEFUL", "Ripper")) {
|
||||
result.addAll(queryByMalware(malware));
|
||||
}
|
||||
|
||||
// Recent IOCs from last 7 days — filter for ATM relevance in caller
|
||||
result.addAll(queryRecent());
|
||||
|
||||
log.info("ThreatFox: fetched {} raw indicators", result.size());
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<ThreatIoc> queryByMalware(String malware) {
|
||||
List<ThreatIoc> out = new ArrayList<>();
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
|
||||
String body = MAPPER.writeValueAsString(Map.of(
|
||||
"query", "iocs_by_malware_family",
|
||||
"malware_family", malware,
|
||||
"limit", 100
|
||||
));
|
||||
|
||||
HttpEntity<String> entity = new HttpEntity<>(body, headers);
|
||||
String response = restTemplate.postForObject(API_URL, entity, String.class);
|
||||
|
||||
out.addAll(parseResponse(response, malware));
|
||||
} catch (Exception e) {
|
||||
log.warn("ThreatFox: error querying malware {}: {}", malware, e.getMessage());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private List<ThreatIoc> queryRecent() {
|
||||
List<ThreatIoc> out = new ArrayList<>();
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
|
||||
String body = MAPPER.writeValueAsString(Map.of(
|
||||
"query", "get_iocs",
|
||||
"days", 7
|
||||
));
|
||||
|
||||
HttpEntity<String> entity = new HttpEntity<>(body, headers);
|
||||
String response = restTemplate.postForObject(API_URL, entity, String.class);
|
||||
|
||||
JsonNode root = MAPPER.readTree(response);
|
||||
if (!"ok".equals(root.path("query_status").asText())) return out;
|
||||
|
||||
for (JsonNode ioc : root.path("data")) {
|
||||
// Only include if tagged or malware family suggests ATM relevance
|
||||
String malwareFamily = ioc.path("malware").asText("").toLowerCase();
|
||||
boolean atmRelated = malwareFamily.contains("atm") || malwareFamily.contains("diebold")
|
||||
|| malwareFamily.contains("ncr") || malwareFamily.contains("ploutus")
|
||||
|| malwareFamily.contains("tyupkin") || malwareFamily.contains("dispenser");
|
||||
if (!atmRelated) continue;
|
||||
|
||||
ThreatIoc mapped = mapIoc(ioc, malwareFamily);
|
||||
if (mapped != null) out.add(mapped);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("ThreatFox: error querying recent IOCs: {}", e.getMessage());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private List<ThreatIoc> parseResponse(String response, String malware) {
|
||||
List<ThreatIoc> out = new ArrayList<>();
|
||||
try {
|
||||
JsonNode root = MAPPER.readTree(response);
|
||||
if (!"ok".equals(root.path("query_status").asText())) return out;
|
||||
for (JsonNode ioc : root.path("data")) {
|
||||
ThreatIoc mapped = mapIoc(ioc, malware);
|
||||
if (mapped != null) out.add(mapped);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("ThreatFox: parse error for {}: {}", malware, e.getMessage());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private ThreatIoc mapIoc(JsonNode ioc, String malware) {
|
||||
String iocType = ioc.path("ioc_type").asText("").toLowerCase();
|
||||
String value = ioc.path("ioc").asText("").trim();
|
||||
if (value.isBlank()) return null;
|
||||
|
||||
ThreatIoc.IocType type = switch (iocType) {
|
||||
case "md5_hash" -> ThreatIoc.IocType.MD5;
|
||||
case "ip:port", "ip" -> {
|
||||
// Strip port if present
|
||||
String ip = value.contains(":") ? value.substring(0, value.lastIndexOf(':')) : value;
|
||||
yield ThreatIoc.IocType.IP;
|
||||
}
|
||||
case "filepath" -> ThreatIoc.IocType.FILE_PATH;
|
||||
case "filename" -> ThreatIoc.IocType.FILENAME;
|
||||
default -> null;
|
||||
};
|
||||
if (type == null) return null;
|
||||
|
||||
// For IP:port, strip the port
|
||||
if ("ip:port".equals(iocType) && value.contains(":")) {
|
||||
value = value.substring(0, value.lastIndexOf(':'));
|
||||
}
|
||||
|
||||
String iocId = ioc.path("id").asText(malware);
|
||||
String confidence = ioc.path("confidence_level").asText("50");
|
||||
int confidenceInt = 50;
|
||||
try { confidenceInt = Integer.parseInt(confidence); } catch (NumberFormatException ignored) {}
|
||||
|
||||
ThreatIoc.ConfidenceLevel level = confidenceInt >= 75
|
||||
? ThreatIoc.ConfidenceLevel.HIGH
|
||||
: confidenceInt >= 50 ? ThreatIoc.ConfidenceLevel.MEDIUM : ThreatIoc.ConfidenceLevel.LOW;
|
||||
|
||||
return ThreatIoc.builder()
|
||||
.iocType(type)
|
||||
.value(value)
|
||||
.description("ThreatFox malware:" + malware)
|
||||
.source(ThreatIoc.IocSource.THREAT_FOX)
|
||||
.sourceRef(iocId)
|
||||
.confidence(level)
|
||||
.expiresAt(Instant.now().plus(180, ChronoUnit.DAYS))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.hiveops.aria.repository;
|
||||
|
||||
import com.hiveops.aria.entity.IocFeedRun;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
public interface IocFeedRunRepository extends JpaRepository<IocFeedRun, Long> {
|
||||
|
||||
Page<IocFeedRun> findAllByOrderByStartedAtDesc(Pageable pageable);
|
||||
|
||||
Optional<IocFeedRun> findTopByFeedNameOrderByStartedAtDesc(String feedName);
|
||||
|
||||
@Query("SELECT COUNT(r) FROM IocFeedRun r WHERE r.feedName = :feedName AND r.status = 'RUNNING'")
|
||||
long countRunning(String feedName);
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package com.hiveops.aria.service;
|
||||
|
||||
import com.hiveops.aria.entity.IocFeedRun;
|
||||
import com.hiveops.aria.entity.ThreatIoc;
|
||||
import com.hiveops.aria.feed.MalwareBazaarFeedClient;
|
||||
import com.hiveops.aria.feed.OtxFeedClient;
|
||||
import com.hiveops.aria.feed.ThreatFoxFeedClient;
|
||||
import com.hiveops.aria.repository.IocFeedRunRepository;
|
||||
import com.hiveops.aria.repository.ThreatIocRepository;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class IocFeedService {
|
||||
|
||||
private final OtxFeedClient otxClient;
|
||||
private final MalwareBazaarFeedClient malwareBazaarClient;
|
||||
private final ThreatFoxFeedClient threatFoxClient;
|
||||
private final ThreatIocRepository iocRepository;
|
||||
private final IocFeedRunRepository feedRunRepository;
|
||||
|
||||
private static final Map<String, ThreatIoc.IocSource> FEED_SOURCES = Map.of(
|
||||
"OTX", ThreatIoc.IocSource.OTX,
|
||||
"MALWARE_BAZAAR", ThreatIoc.IocSource.MALWARE_BAZAAR,
|
||||
"THREAT_FOX", ThreatIoc.IocSource.THREAT_FOX
|
||||
);
|
||||
|
||||
@Scheduled(cron = "${aria.feed.refresh.cron:0 0 3 * * *}")
|
||||
public void scheduledRefresh() {
|
||||
log.info("IOC feed scheduled refresh starting");
|
||||
refreshAll();
|
||||
}
|
||||
|
||||
public void refreshAll() {
|
||||
for (String feedName : FEED_SOURCES.keySet()) {
|
||||
try {
|
||||
refreshFeed(feedName);
|
||||
} catch (Exception e) {
|
||||
log.error("IOC feed {} failed: {}", feedName, e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public IocFeedRun refreshFeed(String feedName) {
|
||||
if (feedRunRepository.countRunning(feedName) > 0) {
|
||||
log.info("IOC feed {} already running, skipping", feedName);
|
||||
return feedRunRepository.findTopByFeedNameOrderByStartedAtDesc(feedName).orElseThrow();
|
||||
}
|
||||
|
||||
IocFeedRun run = feedRunRepository.save(IocFeedRun.builder()
|
||||
.feedName(feedName)
|
||||
.startedAt(Instant.now())
|
||||
.build());
|
||||
|
||||
try {
|
||||
List<ThreatIoc> fetched = fetchFromFeed(feedName);
|
||||
int[] counts = upsertIocs(fetched);
|
||||
|
||||
run.setStatus("SUCCESS");
|
||||
run.setIocsAdded(counts[0]);
|
||||
run.setIocsUpdated(counts[1]);
|
||||
run.setCompletedAt(Instant.now());
|
||||
log.info("IOC feed {} complete: {} added, {} updated", feedName, counts[0], counts[1]);
|
||||
} catch (Exception e) {
|
||||
run.setStatus("ERROR");
|
||||
run.setErrorMessage(e.getMessage());
|
||||
run.setCompletedAt(Instant.now());
|
||||
log.error("IOC feed {} error: {}", feedName, e.getMessage(), e);
|
||||
}
|
||||
|
||||
return feedRunRepository.save(run);
|
||||
}
|
||||
|
||||
private List<ThreatIoc> fetchFromFeed(String feedName) {
|
||||
return switch (feedName) {
|
||||
case "OTX" -> otxClient.fetch();
|
||||
case "MALWARE_BAZAAR" -> malwareBazaarClient.fetch();
|
||||
case "THREAT_FOX" -> threatFoxClient.fetch();
|
||||
default -> throw new IllegalArgumentException("Unknown feed: " + feedName);
|
||||
};
|
||||
}
|
||||
|
||||
private int[] upsertIocs(List<ThreatIoc> iocs) {
|
||||
int added = 0, updated = 0;
|
||||
for (ThreatIoc ioc : iocs) {
|
||||
if (ioc.getValue() == null || ioc.getValue().isBlank()) continue;
|
||||
Optional<ThreatIoc> existing = iocRepository
|
||||
.findByActiveTrueAndIocTypeAndValueIgnoreCase(ioc.getIocType(), ioc.getValue());
|
||||
if (existing.isPresent()) {
|
||||
ThreatIoc e = existing.get();
|
||||
e.setDescription(ioc.getDescription());
|
||||
e.setSourceRef(ioc.getSourceRef());
|
||||
e.setConfidence(ioc.getConfidence());
|
||||
e.setExpiresAt(ioc.getExpiresAt());
|
||||
iocRepository.save(e);
|
||||
updated++;
|
||||
} else {
|
||||
iocRepository.save(ioc);
|
||||
added++;
|
||||
}
|
||||
}
|
||||
return new int[]{ added, updated };
|
||||
}
|
||||
|
||||
public List<String> feedNames() {
|
||||
return List.copyOf(FEED_SOURCES.keySet());
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,11 @@ aria.internal.service-secret=${ARIA_SERVICE_SECRET:}
|
||||
# Sequence detection window (minutes)
|
||||
aria.sequence.window.minutes=${ARIA_SEQUENCE_WINDOW_MINUTES:15}
|
||||
|
||||
# IOC Feed Refresh
|
||||
aria.feed.enabled=${ARIA_FEED_ENABLED:true}
|
||||
aria.feed.refresh.cron=${ARIA_FEED_REFRESH_CRON:0 0 3 * * *}
|
||||
aria.feed.otx.api-key=${ARIA_FEED_OTX_API_KEY:}
|
||||
|
||||
# Auto-response — disabled by default for safe initial deployment
|
||||
aria.autoresponse.shutdown.enabled=${ARIA_AUTORESPONSE_SHUTDOWN_ENABLED:false}
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE ioc_feed_run (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
feed_name VARCHAR(50) NOT NULL,
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'RUNNING',
|
||||
iocs_added INT NOT NULL DEFAULT 0,
|
||||
iocs_updated INT NOT NULL DEFAULT 0,
|
||||
error_message TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_ioc_feed_run_feed_name ON ioc_feed_run (feed_name);
|
||||
CREATE INDEX idx_ioc_feed_run_started_at ON ioc_feed_run (started_at DESC);
|
||||
Reference in New Issue
Block a user