From 89433d782a834ae4ab1835a0b70fa16340bbbd49 Mon Sep 17 00:00:00 2001
From: Junjie <fallin.jie@qq.com>
Date: 星期一, 13 四月 2026 19:41:20 +0800
Subject: [PATCH] #算法耗时优化
---
src/main/java/com/zy/core/task/DeviceLogScheduler.java | 109 ++++++++++++++++++++++++++++++++++++++++++++----------
1 files changed, 89 insertions(+), 20 deletions(-)
diff --git a/src/main/java/com/zy/core/task/DeviceLogScheduler.java b/src/main/java/com/zy/core/task/DeviceLogScheduler.java
index 8f5ff54..c448830 100644
--- a/src/main/java/com/zy/core/task/DeviceLogScheduler.java
+++ b/src/main/java/com/zy/core/task/DeviceLogScheduler.java
@@ -7,6 +7,7 @@
import com.zy.asrs.service.DeviceDataLogService;
import com.zy.common.utils.RedisUtil;
import com.zy.core.enums.RedisKeyType;
+import com.zy.core.enums.SlaveType;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@@ -18,24 +19,29 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
-import java.util.Set;
+import java.util.LinkedHashSet;
import java.util.List;
-import java.util.ArrayList;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Stream;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
@Slf4j
@Component
public class DeviceLogScheduler {
+
+ private static final int BASE_BATCH_SIZE = 100;
+ private static final int BACKLOG_SCAN_LIMIT = 2000;
+ private static final int MAX_BATCH_SIZE = 1000;
@Value("${deviceLogStorage.type}")
private String storageType;
@@ -54,7 +60,7 @@
public void delDeviceLog() {
if ("mysql".equals(storageType)) {
deviceDataLogService.clearLog(expireDays == null ? 1 : expireDays);
- }else if ("file".equals(storageType)) {
+ } else if ("file".equals(storageType)) {
if (!FILE_OP_LOCK.tryLock()) {
return;
}
@@ -63,18 +69,18 @@
} finally {
FILE_OP_LOCK.unlock();
}
- }else {
+ } else {
log.error("鏈畾涔夌殑瀛樺偍绫诲瀷锛歿}", storageType);
}
}
@Scheduled(cron = "0/3 * * * * ? ")
public void execute() {
- int maxCount = 100;
- Set<String> keys = redisUtil.scanKeys(RedisKeyType.DEVICE_LOG_KEY.key, maxCount);
- if (keys == null || keys.isEmpty()) {
+ Set<String> scannedKeys = redisUtil.scanKeys(RedisKeyType.DEVICE_LOG_KEY.key, BACKLOG_SCAN_LIMIT);
+ if (scannedKeys == null || scannedKeys.isEmpty()) {
return;
}
+ Set<String> keys = selectBatchKeys(scannedKeys);
List<Object> values = redisUtil.multiGet(keys);
List<DeviceDataLog> list = new ArrayList<>();
for (Object object : values) {
@@ -85,7 +91,7 @@
if (!list.isEmpty()) {
if ("mysql".equals(storageType)) {
mysqlSave(keys, list);
- }else if ("file".equals(storageType)) {
+ } else if ("file".equals(storageType)) {
if (!FILE_OP_LOCK.tryLock()) {
return;
}
@@ -94,10 +100,35 @@
} finally {
FILE_OP_LOCK.unlock();
}
- }else {
+ } else {
log.error("鏈畾涔夌殑瀛樺偍绫诲瀷锛歿}", storageType);
}
}
+ }
+
+ private Set<String> selectBatchKeys(Set<String> scannedKeys) {
+ int backlog = scannedKeys.size();
+ int batchSize = resolveBatchSize(backlog);
+ if (backlog <= batchSize) {
+ return scannedKeys;
+ }
+ LinkedHashSet<String> selected = new LinkedHashSet<>();
+ for (String key : scannedKeys) {
+ selected.add(key);
+ if (selected.size() >= batchSize) {
+ break;
+ }
+ }
+ return selected;
+ }
+
+ private int resolveBatchSize(int backlog) {
+ if (backlog <= BASE_BATCH_SIZE) {
+ return backlog;
+ }
+ int adaptive = Math.max(BASE_BATCH_SIZE, backlog / 2);
+ int rounded = ((adaptive + BASE_BATCH_SIZE - 1) / BASE_BATCH_SIZE) * BASE_BATCH_SIZE;
+ return Math.min(MAX_BATCH_SIZE, rounded);
}
private void mysqlSave(Set<String> keys, List<DeviceDataLog> list) {
@@ -113,9 +144,11 @@
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
Map<String, Map<String, List<DeviceDataLog>>> group = new HashMap<>();
for (DeviceDataLog logItem : list) {
- String typeName = logItem.getType();
String datePart = sdf.format(logItem.getCreateTime() == null ? new Date() : logItem.getCreateTime());
- String prefix = typeName + "_" + String.valueOf(logItem.getDeviceNo()) + "_" + datePart + "_";
+ String prefix = buildFilePrefix(logItem, datePart);
+ if (prefix == null) {
+ continue;
+ }
group.computeIfAbsent(datePart, k -> new HashMap<>())
.computeIfAbsent(prefix, k -> new ArrayList<>())
.add(logItem);
@@ -124,11 +157,23 @@
Path dayDir = baseDir.resolve(dateEntry.getKey());
Files.createDirectories(dayDir);
for (Map.Entry<String, List<DeviceDataLog>> entry : dateEntry.getValue().entrySet()) {
- String prefix = entry.getKey();
List<DeviceDataLog> logs = entry.getValue();
+ if (logs == null || logs.isEmpty()) {
+ continue;
+ }
+ DeviceDataLog firstLog = logs.get(0);
+ Path deviceDir = resolveDeviceDir(dayDir, firstLog);
+ if (deviceDir == null) {
+ continue;
+ }
+ Files.createDirectories(deviceDir);
+ String prefix = buildFilePrefix(firstLog, dateEntry.getKey());
+ if (prefix == null) {
+ continue;
+ }
logs.sort(Comparator.comparing(DeviceDataLog::getCreateTime, Comparator.nullsLast(Date::compareTo)));
- int index = findStartIndex(dayDir, prefix);
- Path current = dayDir.resolve(prefix + index + ".log");
+ int index = findStartIndex(deviceDir, prefix);
+ Path current = deviceDir.resolve(prefix + index + ".log");
if (!Files.exists(current)) {
Files.createFile(current);
}
@@ -139,7 +184,7 @@
byte[] line = (json + System.lineSeparator()).getBytes(StandardCharsets.UTF_8);
if (size + line.length > max) {
index++;
- current = dayDir.resolve(prefix + index + ".log");
+ current = deviceDir.resolve(prefix + index + ".log");
if (!Files.exists(current)) {
Files.createFile(current);
}
@@ -154,6 +199,27 @@
} catch (Exception e) {
log.error("璁惧鏃ュ織鏂囦欢瀛樺偍澶辫触", e);
}
+ }
+
+ private String buildFilePrefix(DeviceDataLog logItem, String datePart) {
+ if (logItem == null || logItem.getType() == null || logItem.getDeviceNo() == null) {
+ return null;
+ }
+ if (String.valueOf(SlaveType.Devp).equals(logItem.getType())) {
+ if (logItem.getStationId() == null) {
+ log.warn("璺宠繃缂哄皯绔欑偣鍙风殑杈撻�佽澶囨棩蹇�, deviceNo={}, createTime={}", logItem.getDeviceNo(), logItem.getCreateTime());
+ return null;
+ }
+ return logItem.getType() + "_" + logItem.getDeviceNo() + "_station_" + logItem.getStationId() + "_" + datePart + "_";
+ }
+ return logItem.getType() + "_" + logItem.getDeviceNo() + "_" + datePart + "_";
+ }
+
+ private Path resolveDeviceDir(Path dayDir, DeviceDataLog logItem) {
+ if (dayDir == null || logItem == null || logItem.getType() == null || logItem.getDeviceNo() == null) {
+ return null;
+ }
+ return dayDir.resolve(logItem.getType()).resolve(String.valueOf(logItem.getDeviceNo()));
}
private int findStartIndex(Path baseDir, String prefix) throws Exception {
@@ -176,7 +242,8 @@
if (val > maxIdx) {
maxIdx = val;
}
- } catch (NumberFormatException ignored) {}
+ } catch (NumberFormatException ignored) {
+ }
}
}
int candidate = maxIdx == 0 ? 1 : maxIdx;
@@ -212,7 +279,8 @@
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
try {
Files.deleteIfExists(file);
- } catch (Exception ignored) {}
+ } catch (Exception ignored) {
+ }
return FileVisitResult.CONTINUE;
}
@@ -220,7 +288,8 @@
public FileVisitResult postVisitDirectory(Path dir, java.io.IOException exc) {
try {
Files.deleteIfExists(dir);
- } catch (Exception ignored) {}
+ } catch (Exception ignored) {
+ }
return FileVisitResult.CONTINUE;
}
});
--
Gitblit v1.9.1