package com.zy.core.task;
|
|
|
import com.alibaba.fastjson.JSON;
|
import com.alibaba.fastjson.serializer.SerializerFeature;
|
import com.zy.asrs.entity.DeviceDataLog;
|
import com.zy.asrs.service.DeviceDataLogService;
|
import com.zy.common.utils.RedisUtil;
|
import com.zy.core.enums.RedisKeyType;
|
import lombok.extern.slf4j.Slf4j;
|
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.stereotype.Component;
|
|
import java.nio.charset.StandardCharsets;
|
import java.nio.file.FileVisitResult;
|
import java.nio.file.Files;
|
import java.nio.file.Path;
|
import java.nio.file.Paths;
|
import java.nio.file.StandardOpenOption;
|
import java.nio.file.SimpleFileVisitor;
|
import java.nio.file.attribute.BasicFileAttributes;
|
import java.text.SimpleDateFormat;
|
import java.util.Comparator;
|
import java.util.Date;
|
import java.util.HashMap;
|
import java.util.LinkedHashSet;
|
import java.util.Set;
|
import java.util.List;
|
import java.util.ArrayList;
|
import java.util.Map;
|
import java.util.concurrent.locks.ReentrantLock;
|
import java.util.stream.Stream;
|
import java.util.stream.Collectors;
|
|
@Slf4j
|
@Component
|
public class DeviceLogScheduler {
|
|
private static final int BASE_BATCH_SIZE = 100;
|
private static final int BACKLOG_SCAN_LIMIT = 2000;
|
private static final int MAX_BATCH_SIZE = 1000;
|
|
@Value("${deviceLogStorage.type}")
|
private String storageType;
|
@Value("${deviceLogStorage.loggingPath}")
|
private String loggingPath;
|
@Value("${deviceLogStorage.expireDays}")
|
private Integer expireDays;
|
@Autowired
|
private DeviceDataLogService deviceDataLogService;
|
@Autowired
|
private RedisUtil redisUtil;
|
|
private static final ReentrantLock FILE_OP_LOCK = new ReentrantLock();
|
|
@Scheduled(cron = "0/3 * * * * ? ")
|
public void delDeviceLog() {
|
if ("mysql".equals(storageType)) {
|
deviceDataLogService.clearLog(expireDays == null ? 1 : expireDays);
|
}else if ("file".equals(storageType)) {
|
if (!FILE_OP_LOCK.tryLock()) {
|
return;
|
}
|
try {
|
clearFileLog(expireDays == null ? 1 : expireDays);
|
} finally {
|
FILE_OP_LOCK.unlock();
|
}
|
}else {
|
log.error("未定义的存储类型：{}", storageType);
|
}
|
}
|
|
@Scheduled(cron = "0/3 * * * * ? ")
|
public void execute() {
|
Set<String> scannedKeys = redisUtil.scanKeys(RedisKeyType.DEVICE_LOG_KEY.key, BACKLOG_SCAN_LIMIT);
|
if (scannedKeys == null || scannedKeys.isEmpty()) {
|
return;
|
}
|
Set<String> keys = selectBatchKeys(scannedKeys);
|
List<Object> values = redisUtil.multiGet(keys);
|
List<DeviceDataLog> list = new ArrayList<>();
|
for (Object object : values) {
|
if (object instanceof DeviceDataLog) {
|
list.add((DeviceDataLog) object);
|
}
|
}
|
if (!list.isEmpty()) {
|
if ("mysql".equals(storageType)) {
|
mysqlSave(keys, list);
|
}else if ("file".equals(storageType)) {
|
if (!FILE_OP_LOCK.tryLock()) {
|
return;
|
}
|
try {
|
fileSave(keys, list);
|
} finally {
|
FILE_OP_LOCK.unlock();
|
}
|
}else {
|
log.error("未定义的存储类型：{}", storageType);
|
}
|
}
|
}
|
|
private Set<String> selectBatchKeys(Set<String> scannedKeys) {
|
int backlog = scannedKeys.size();
|
int batchSize = resolveBatchSize(backlog);
|
if (backlog <= batchSize) {
|
return scannedKeys;
|
}
|
LinkedHashSet<String> selected = new LinkedHashSet<>();
|
for (String key : scannedKeys) {
|
selected.add(key);
|
if (selected.size() >= batchSize) {
|
break;
|
}
|
}
|
return selected;
|
}
|
|
private int resolveBatchSize(int backlog) {
|
if (backlog <= BASE_BATCH_SIZE) {
|
return backlog;
|
}
|
int adaptive = Math.max(BASE_BATCH_SIZE, backlog / 2);
|
int rounded = ((adaptive + BASE_BATCH_SIZE - 1) / BASE_BATCH_SIZE) * BASE_BATCH_SIZE;
|
return Math.min(MAX_BATCH_SIZE, rounded);
|
}
|
|
private void mysqlSave(Set<String> keys, List<DeviceDataLog> list) {
|
if (deviceDataLogService.saveBatch(list)) {
|
redisUtil.del(keys.toArray(new String[0]));
|
}
|
}
|
|
private void fileSave(Set<String> keys, List<DeviceDataLog> list) {
|
try {
|
Path baseDir = Paths.get(loggingPath);
|
Files.createDirectories(baseDir);
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
|
Map<String, Map<String, List<DeviceDataLog>>> group = new HashMap<>();
|
for (DeviceDataLog logItem : list) {
|
String typeName = logItem.getType();
|
String datePart = sdf.format(logItem.getCreateTime() == null ? new Date() : logItem.getCreateTime());
|
String prefix = typeName + "_" + String.valueOf(logItem.getDeviceNo()) + "_" + datePart + "_";
|
group.computeIfAbsent(datePart, k -> new HashMap<>())
|
.computeIfAbsent(prefix, k -> new ArrayList<>())
|
.add(logItem);
|
}
|
for (Map.Entry<String, Map<String, List<DeviceDataLog>>> dateEntry : group.entrySet()) {
|
Path dayDir = baseDir.resolve(dateEntry.getKey());
|
Files.createDirectories(dayDir);
|
for (Map.Entry<String, List<DeviceDataLog>> entry : dateEntry.getValue().entrySet()) {
|
String prefix = entry.getKey();
|
List<DeviceDataLog> logs = entry.getValue();
|
logs.sort(Comparator.comparing(DeviceDataLog::getCreateTime, Comparator.nullsLast(Date::compareTo)));
|
int index = findStartIndex(dayDir, prefix);
|
Path current = dayDir.resolve(prefix + index + ".log");
|
if (!Files.exists(current)) {
|
Files.createFile(current);
|
}
|
long size = Files.size(current);
|
long max = 1024L * 1024L;
|
for (DeviceDataLog d : logs) {
|
String json = JSON.toJSONStringWithDateFormat(d, "yyyy-MM-dd HH:mm:ss.SSS", SerializerFeature.WriteDateUseDateFormat);
|
byte[] line = (json + System.lineSeparator()).getBytes(StandardCharsets.UTF_8);
|
if (size + line.length > max) {
|
index++;
|
current = dayDir.resolve(prefix + index + ".log");
|
if (!Files.exists(current)) {
|
Files.createFile(current);
|
}
|
size = 0;
|
}
|
Files.write(current, line, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
size += line.length;
|
}
|
}
|
}
|
redisUtil.del(keys.toArray(new String[0]));
|
} catch (Exception e) {
|
log.error("设备日志文件存储失败", e);
|
}
|
}
|
|
private int findStartIndex(Path baseDir, String prefix) throws Exception {
|
List<Path> matched;
|
try (Stream<Path> stream = Files.list(baseDir)) {
|
matched = stream
|
.filter(p -> {
|
String n = p.getFileName().toString();
|
return n.startsWith(prefix) && n.endsWith(".log");
|
})
|
.collect(Collectors.toList());
|
}
|
int maxIdx = 0;
|
for (Path p : matched) {
|
String name = p.getFileName().toString();
|
String suf = name.substring(prefix.length());
|
if (!suf.isEmpty()) {
|
try {
|
int val = Integer.parseInt(suf.replace(".log", ""));
|
if (val > maxIdx) {
|
maxIdx = val;
|
}
|
} catch (NumberFormatException ignored) {}
|
}
|
}
|
int candidate = maxIdx == 0 ? 1 : maxIdx;
|
Path path = baseDir.resolve(prefix + candidate + ".log");
|
if (Files.exists(path)) {
|
long size = Files.size(path);
|
if (size >= 1024L * 1024L) {
|
return candidate + 1;
|
}
|
}
|
return candidate;
|
}
|
|
private void clearFileLog(int days) {
|
try {
|
Path baseDir = Paths.get(loggingPath);
|
if (!Files.exists(baseDir)) {
|
return;
|
}
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
|
long cutoff = System.currentTimeMillis() - (long) days * 24 * 60 * 60 * 1000;
|
List<Path> dirs;
|
try (Stream<Path> stream = Files.list(baseDir)) {
|
dirs = stream.filter(Files::isDirectory).collect(Collectors.toList());
|
}
|
for (Path dir : dirs) {
|
String name = dir.getFileName().toString();
|
if (name.length() == 8 && name.chars().allMatch(Character::isDigit)) {
|
Date d = sdf.parse(name);
|
if (d.getTime() < cutoff) {
|
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
|
@Override
|
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
try {
|
Files.deleteIfExists(file);
|
} catch (Exception ignored) {}
|
return FileVisitResult.CONTINUE;
|
}
|
|
@Override
|
public FileVisitResult postVisitDirectory(Path dir, java.io.IOException exc) {
|
try {
|
Files.deleteIfExists(dir);
|
} catch (Exception ignored) {}
|
return FileVisitResult.CONTINUE;
|
}
|
});
|
}
|
}
|
}
|
} catch (Exception e) {
|
log.error("设备日志文件清理失败", e);
|
}
|
}
|
|
}
|
