#real connect remove write data V3.0.2.0

Junjie

1 天以前 13b2678b0d6402cefee85bafe4da69aa72528a28

 src/main/java/com/zy/asrs/controller/DeviceLogController.java

@@ -18,14 +18,21 @@
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import jakarta.servlet.http.HttpServletResponse;
import java.io.ByteArrayOutputStream;
import java.io.RandomAccessFile;
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
@@ -50,6 +57,15 @@

    @Value("${deviceLogStorage.loggingPath}")
    private String loggingPath;

    @Value("${logging.file.path:./stock/out/@pom.build.finalName@/logs}")
    private String systemLoggingPath;

    private static final DateTimeFormatter SYSTEM_LOG_DATE = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    private static final DateTimeFormatter SYSTEM_LOG_DATE_TIME = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
    private static final DateTimeFormatter SYSTEM_LOG_EXPORT_TIME = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss");
    private static final Pattern SYSTEM_LOG_TIMESTAMP = Pattern.compile("^(\\d{2}:\\d{2}:\\d{2}\\.\\d{3})");
    private static final Pattern SYSTEM_LOG_ROLLED_FILE = Pattern.compile("^(info|error)_(\\d{4}-\\d{2}-\\d{2})\\.(\\d+)\\.log$");

    private static class ProgressInfo {
        long totalRaw;
@@ -87,7 +103,27 @@
        Path lastFile;
    }

    private static class SystemLogFileInfo {
        String logType;
        LocalDate day;
        Integer index;
        boolean active;
        Path path;
    }

    private static class SystemLogDownloadRequest {
        String logType;
        LocalDateTime startTime;
        LocalDateTime endTime;
        List<Path> files;
    }

    private static final Map<String, ProgressInfo> DOWNLOAD_PROGRESS = new ConcurrentHashMap<>();
    private static final int SYSTEM_LOG_MAX_RANGE_DAYS = 10;
    private static final int SYSTEM_LOG_MATCH_BUFFER_LINES = 200000;
    private static final String SYSTEM_LOG_ENTRY_NAME = "system.log";

    private static final Map<String, SystemLogDownloadRequest> SYSTEM_LOG_DOWNLOAD_REQUESTS = new ConcurrentHashMap<>();

    @RequestMapping(value = "/deviceLog/dates/auth")
    @ManagerAuth
@@ -226,11 +262,11 @@
            }
            for (DeviceAggregate aggregate : aggregateMap.values()) {
                if (aggregate.firstFile != null) {
                    FileTimeRange firstRange = readFileTimeRange(aggregate.firstFile);
                    FileTimeRange firstRange = readFileTimeRange(aggregate.firstFile, aggregate.stationId);
                    aggregate.firstTime = firstRange.startTime != null ? firstRange.startTime : firstRange.endTime;
                }
                if (aggregate.lastFile != null) {
                    FileTimeRange lastRange = readFileTimeRange(aggregate.lastFile);
                    FileTimeRange lastRange = readFileTimeRange(aggregate.lastFile, aggregate.stationId);
                    aggregate.lastTime = lastRange.endTime != null ? lastRange.endTime : lastRange.startTime;
                }
            }
@@ -274,7 +310,7 @@
            List<Map<String, Object>> segments = new ArrayList<>();
            Long startTime = null;
            for (int i = 0; i < files.size(); i++) {
                Long segmentStart = getFileStartTime(files.get(i));
                Long segmentStart = getFileStartTime(files.get(i), stationId);
                if (segmentStart != null && (startTime == null || segmentStart < startTime)) {
                    startTime = segmentStart;
                }
@@ -284,7 +320,7 @@
                segment.put("endTime", null);
                segments.add(segment);
            }
            Long endTime = getFileEndTime(files.get(files.size() - 1));
            Long endTime = getFileEndTime(files.get(files.size() - 1), stationId);
            if (endTime == null) {
                for (int i = segments.size() - 1; i >= 0; i--) {
                    Long segmentStart = (Long) segments.get(i).get("startTime");
@@ -370,7 +406,9 @@
                        if (line != null && !line.trim().isEmpty()) {
                            try {
                                DeviceDataLog logItem = JSON.parseObject(line, DeviceDataLog.class);
                                resultLogs.add(logItem);
                                if (matchesRequestedStation(logItem, stationId)) {
                                    resultLogs.add(logItem);
                                }
                            } catch (Exception e) {
                            }
                        }
@@ -427,7 +465,7 @@
            while (low <= high) {
                int mid = (low + high) >>> 1;
                Path midFile = files.get(mid);
                Long midStart = getFileStartTime(midFile);
                Long midStart = getFileStartTime(midFile, stationId);
                if (midStart == null) {
                    low = mid + 1;
                    continue;
@@ -455,9 +493,9 @@
        }
    }
    
    private Long getFileStartTime(Path file) {
    private Long getFileStartTime(Path file, String stationId) {
        try {
            String firstLine = readFirstNonBlankLine(file);
            String firstLine = readFirstMatchingLine(file, stationId);
            if (firstLine == null) return null;
            DeviceDataLog firstLog = JSON.parseObject(firstLine, DeviceDataLog.class);
            return firstLog.getCreateTime().getTime();
@@ -466,9 +504,9 @@
        }
    }

    private Long getFileEndTime(Path file) {
    private Long getFileEndTime(Path file, String stationId) {
        try {
            String lastLine = readLastNonBlankLine(file);
            String lastLine = readLastMatchingLine(file, stationId);
            if (lastLine == null) return null;
            DeviceDataLog lastLog = JSON.parseObject(lastLine, DeviceDataLog.class);
            return lastLog.getCreateTime().getTime();
@@ -651,20 +689,145 @@
        return R.ok(res);
    }

    @RequestMapping(value = "/deviceLog/system/download/init/auth")
    @ManagerAuth
    public R initSystemDownload(@org.springframework.web.bind.annotation.RequestBody com.alibaba.fastjson.JSONObject param) {
        try {
            String logType = normalizeSystemLogType(param.getString("logType"));
            if (logType == null) {
                return R.error("日志类型错误");
            }
            LocalDateTime startTime = parseSystemLogDateTime(param.getString("startTime"));
            LocalDateTime endTime = parseSystemLogDateTime(param.getString("endTime"));
            if (startTime == null || endTime == null) {
                return R.error("时间格式错误");
            }
            if (startTime.isAfter(endTime)) {
                return R.error("开始时间不能晚于结束时间");
            }
            long daySpan = java.time.temporal.ChronoUnit.DAYS.between(startTime.toLocalDate(), endTime.toLocalDate());
            if (daySpan > SYSTEM_LOG_MAX_RANGE_DAYS) {
                return R.error("时间范围不能超过" + SYSTEM_LOG_MAX_RANGE_DAYS + "天");
            }
            List<Path> files = findSystemLogFiles(logType, startTime, endTime);
            if (files.isEmpty()) {
                return R.error("未找到日志文件");
            }
            String id = UUID.randomUUID().toString();
            ProgressInfo info = new ProgressInfo();
            info.totalCount = files.size();
            long sum = 0L;
            for (Path f : files) {
                try { sum += Files.size(f); } catch (Exception ignored) {}
            }
            info.totalRaw = sum;
            info.processedRaw = 0L;
            info.processedCount = 0;
            info.finished = false;
            DOWNLOAD_PROGRESS.put(id, info);
            SystemLogDownloadRequest request = new SystemLogDownloadRequest();
            request.logType = logType;
            request.startTime = startTime;
            request.endTime = endTime;
            request.files = files;
            SYSTEM_LOG_DOWNLOAD_REQUESTS.put(id, request);
            Map<String, Object> res = new HashMap<>();
            res.put("progressId", id);
            res.put("totalSize", info.totalRaw);
            res.put("fileCount", info.totalCount);
            return R.ok(res);
        } catch (Exception e) {
            log.error("初始化系统日志下载失败", e);
            return R.error("初始化失败");
        }
    }

    @RequestMapping(value = "/deviceLog/system/download/auth")
    @ManagerAuth
    public void downloadSystemLog(@RequestParam("logType") String logTypeParam,
                                  @RequestParam("startTime") String startTimeParam,
                                  @RequestParam("endTime") String endTimeParam,
                                  @RequestParam(value = "progressId", required = false) String progressId,
                                  HttpServletResponse response) {
        String progressKey = null;
        try {
            String logType = normalizeSystemLogType(logTypeParam);
            LocalDateTime startTime = parseSystemLogDateTime(startTimeParam);
            LocalDateTime endTime = parseSystemLogDateTime(endTimeParam);
            if (logType == null || startTime == null || endTime == null || startTime.isAfter(endTime)) {
                response.setStatus(400);
                return;
            }
            SystemLogDownloadRequest requestInfo = null;
            if (!Cools.isEmpty(progressId)) {
                requestInfo = SYSTEM_LOG_DOWNLOAD_REQUESTS.get(progressId);
                progressKey = progressId;
            }
            List<Path> files;
            if (requestInfo != null
                    && Objects.equals(requestInfo.logType, logType)
                    && Objects.equals(requestInfo.startTime, startTime)
                    && Objects.equals(requestInfo.endTime, endTime)) {
                files = requestInfo.files == null ? Collections.emptyList() : requestInfo.files;
            } else {
                files = findSystemLogFiles(logType, startTime, endTime);
            }
            if (files.isEmpty()) {
                response.setStatus(404);
                return;
            }
            if (Cools.isEmpty(progressKey)) {
                progressKey = UUID.randomUUID().toString();
            }
            ProgressInfo info = prepareProgress(progressKey, files);
            response.reset();
            response.setContentType("application/zip");
            String filename = logType + "_" + formatSystemExportTime(startTime) + "_" + formatSystemExportTime(endTime) + ".zip";
            response.setHeader("Content-Disposition", "attachment; filename=" + filename);
            response.setHeader("X-Progress-Id", progressKey);
            try (ZipOutputStream zos = new ZipOutputStream(response.getOutputStream())) {
                zos.putNextEntry(new ZipEntry(SYSTEM_LOG_ENTRY_NAME));
                try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zos, StandardCharsets.UTF_8))) {
                    boolean written = writeSystemLogContent(files, logType, startTime, endTime, writer, info);
                    writer.flush();
                    if (!written) {
                        response.reset();
                        response.setStatus(404);
                        return;
                    }
                }
                zos.closeEntry();
                zos.finish();
                info.finished = true;
            }
        } catch (Exception e) {
            log.error("下载系统日志失败", e);
            try {
                response.reset();
                response.setStatus(500);
            } catch (Exception ignore) {
            }
        } finally {
            if (!Cools.isEmpty(progressId)) {
                SYSTEM_LOG_DOWNLOAD_REQUESTS.remove(progressId);
            }
        }
    }

    @RequestMapping(value = "/deviceLog/enums/auth")
    @ManagerAuth
    public R getEnums() {
        Map<String, Map<String, String>> enums = new HashMap<>();
        

        enums.put("CrnModeType", Arrays.stream(com.zy.core.enums.CrnModeType.values())
                .collect(Collectors.toMap(e -> String.valueOf(e.id), e -> e.desc)));
        

        enums.put("CrnStatusType", Arrays.stream(com.zy.core.enums.CrnStatusType.values())
                .collect(Collectors.toMap(e -> String.valueOf(e.id), e -> e.desc)));
        

        enums.put("CrnForkPosType", Arrays.stream(com.zy.core.enums.CrnForkPosType.values())
                .collect(Collectors.toMap(e -> String.valueOf(e.id), e -> e.desc)));
        

        enums.put("CrnLiftPosType", Arrays.stream(com.zy.core.enums.CrnLiftPosType.values())
                .collect(Collectors.toMap(e -> String.valueOf(e.id), e -> e.desc)));

@@ -679,9 +842,210 @@

        enums.put("RgvStatusType", Arrays.stream(com.zy.core.enums.RgvStatusType.values())
                .collect(Collectors.toMap(e -> String.valueOf(e.id), e -> e.desc)));
        

        return R.ok(enums);
    }

    private ProgressInfo prepareProgress(String progressId, List<Path> files) {
        return DOWNLOAD_PROGRESS.compute(progressId, (key, existing) -> {
            ProgressInfo next = existing == null ? new ProgressInfo() : existing;
            next.totalCount = files == null ? 0 : files.size();
            long total = 0L;
            if (files != null) {
                for (Path f : files) {
                    try { total += Files.size(f); } catch (Exception ignored) {}
                }
            }
            next.totalRaw = total;
            next.processedRaw = 0L;
            next.processedCount = 0;
            next.finished = false;
            return next;
        });
    }

    private boolean writeSystemLogContent(List<Path> files,
                                          String logType,
                                          LocalDateTime startTime,
                                          LocalDateTime endTime,
                                          BufferedWriter writer,
                                          ProgressInfo progressInfo) throws Exception {
        boolean wroteAny = false;
        for (Path file : files) {
            SystemLogFileInfo fileInfo = parseSystemLogFile(file);
            if (fileInfo == null) {
                updateProgress(progressInfo, file);
                continue;
            }
            LocalDate baseDate = fileInfo.day != null ? fileInfo.day : startTime.toLocalDate();
            try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
                List<String> pending = new ArrayList<>();
                boolean pendingMatched = false;
                for (Iterator<String> iterator = lines.iterator(); iterator.hasNext(); ) {
                    String line = iterator.next();
                    Matcher matcher = SYSTEM_LOG_TIMESTAMP.matcher(line == null ? "" : line);
                    if (matcher.find()) {
                        if (pendingMatched && !pending.isEmpty()) {
                            for (String pendingLine : pending) {
                                writer.write(pendingLine == null ? "" : pendingLine);
                                writer.newLine();
                            }
                            wroteAny = true;
                        }
                        pending.clear();
                        pendingMatched = isSystemLogLineInRange(baseDate, matcher.group(1), startTime, endTime);
                    }
                    if (pendingMatched) {
                        pending.add(line);
                        if (pending.size() > SYSTEM_LOG_MATCH_BUFFER_LINES) {
                            for (String pendingLine : pending) {
                                writer.write(pendingLine == null ? "" : pendingLine);
                                writer.newLine();
                            }
                            wroteAny = true;
                            pending.clear();
                        }
                    } else if (!pending.isEmpty()) {
                        pending.clear();
                    }
                }
                if (pendingMatched && !pending.isEmpty()) {
                    for (String pendingLine : pending) {
                        writer.write(pendingLine == null ? "" : pendingLine);
                        writer.newLine();
                    }
                    wroteAny = true;
                }
            }
            updateProgress(progressInfo, file);
        }
        if (progressInfo != null) {
            progressInfo.finished = true;
        }
        return wroteAny;
    }

    private void updateProgress(ProgressInfo progressInfo, Path file) {
        if (progressInfo == null) {
            return;
        }
        try {
            progressInfo.processedRaw += Files.size(file);
        } catch (Exception ignored) {
        }
        progressInfo.processedCount += 1;
    }

    private List<Path> findSystemLogFiles(String logType, LocalDateTime startTime, LocalDateTime endTime) throws Exception {
        Path baseDir = Paths.get(systemLoggingPath);
        if (!Files.exists(baseDir) || !Files.isDirectory(baseDir)) {
            return Collections.emptyList();
        }
        LocalDate startDate = startTime.toLocalDate();
        LocalDate endDate = endTime.toLocalDate();
        List<SystemLogFileInfo> matched = new ArrayList<>();
        try (Stream<Path> stream = Files.list(baseDir)) {
            stream.filter(path -> !Files.isDirectory(path)).forEach(path -> {
                SystemLogFileInfo info = parseSystemLogFile(path);
                if (info == null || !Objects.equals(info.logType, logType)) {
                    return;
                }
                if (info.active || info.day == null || (!info.day.isBefore(startDate) && !info.day.isAfter(endDate))) {
                    matched.add(info);
                }
            });
        }
        matched.sort((left, right) -> {
            LocalDate leftDay = left.day == null ? LocalDate.MAX : left.day;
            LocalDate rightDay = right.day == null ? LocalDate.MAX : right.day;
            int cmp = leftDay.compareTo(rightDay);
            if (cmp != 0) {
                return cmp;
            }
            int leftIndex = left.index == null ? Integer.MAX_VALUE : left.index;
            int rightIndex = right.index == null ? Integer.MAX_VALUE : right.index;
            if (left.active != right.active) {
                return left.active ? 1 : -1;
            }
            return Integer.compare(leftIndex, rightIndex);
        });
        return matched.stream().map(item -> item.path).collect(Collectors.toList());
    }

    private SystemLogFileInfo parseSystemLogFile(Path path) {
        if (path == null) {
            return null;
        }
        String name = path.getFileName().toString();
        if ("info.log".equals(name) || "error.log".equals(name)) {
            SystemLogFileInfo info = new SystemLogFileInfo();
            info.logType = name.startsWith("info") ? "info" : "error";
            info.active = true;
            info.path = path;
            return info;
        }
        Matcher matcher = SYSTEM_LOG_ROLLED_FILE.matcher(name);
        if (!matcher.matches()) {
            return null;
        }
        try {
            SystemLogFileInfo info = new SystemLogFileInfo();
            info.logType = matcher.group(1);
            info.day = LocalDate.parse(matcher.group(2), SYSTEM_LOG_DATE);
            info.index = Integer.parseInt(matcher.group(3));
            info.active = false;
            info.path = path;
            return info;
        } catch (Exception e) {
            return null;
        }
    }

    private String normalizeSystemLogType(String logType) {
        if (Cools.isEmpty(logType)) {
            return null;
        }
        String value = logType.trim().toLowerCase(Locale.ROOT);
        if ("info".equals(value) || "error".equals(value)) {
            return value;
        }
        return null;
    }

    private LocalDateTime parseSystemLogDateTime(String value) {
        if (Cools.isEmpty(value)) {
            return null;
        }
        try {
            return LocalDateTime.parse(value.trim(), SYSTEM_LOG_DATE_TIME);
        } catch (DateTimeParseException e) {
            return null;
        }
    }

    private boolean isSystemLogLineInRange(LocalDate baseDate,
                                           String timePart,
                                           LocalDateTime startTime,
                                           LocalDateTime endTime) {
        if (baseDate == null || Cools.isEmpty(timePart) || startTime == null || endTime == null) {
            return false;
        }
        try {
            LocalTime time = LocalTime.parse(timePart);
            LocalDateTime timestamp = LocalDateTime.of(baseDate, time);
            return !timestamp.isBefore(startTime) && !timestamp.isAfter(endTime);
        } catch (DateTimeParseException e) {
            return false;
        }
    }

    private String formatSystemExportTime(LocalDateTime dateTime) {
        if (dateTime == null) {
            return "unknown";
        }
        return SYSTEM_LOG_EXPORT_TIME.format(dateTime);
    }


    private Map<String, Object> buildEmptySummary() {
        return buildSummaryResponse(Collections.emptyList());
@@ -943,65 +1307,77 @@
        }
    }

    private FileTimeRange readFileTimeRange(Path file) {
    private FileTimeRange readFileTimeRange(Path file, String stationId) {
        FileTimeRange range = new FileTimeRange();
        try {
            String firstLine = readFirstNonBlankLine(file);
            String lastLine = readLastNonBlankLine(file);
            range.startTime = parseLogTime(firstLine);
            range.endTime = parseLogTime(lastLine);
            String firstLine = readFirstMatchingLine(file, stationId);
            String lastLine = readLastMatchingLine(file, stationId);
            range.startTime = parseLogTime(firstLine, stationId);
            range.endTime = parseLogTime(lastLine, stationId);
            return range;
        } catch (Exception e) {
            return range;
        }
    }

    private Long parseLogTime(String line) {
    private Long parseLogTime(String line, String stationId) {
        try {
            if (line == null || line.trim().isEmpty()) {
                return null;
            }
            DeviceDataLog logItem = JSON.parseObject(line, DeviceDataLog.class);
            if (!matchesRequestedStation(logItem, stationId)) {
                return null;
            }
            return logItem != null && logItem.getCreateTime() != null ? logItem.getCreateTime().getTime() : null;
        } catch (Exception e) {
            return null;
        }
    }

    private String readFirstNonBlankLine(Path file) {
    private boolean matchesRequestedStation(DeviceDataLog logItem, String stationId) {
        if (Cools.isEmpty(stationId)) {
            return true;
        }
        if (logItem == null || logItem.getStationId() == null) {
            return false;
        }
        return Objects.equals(String.valueOf(logItem.getStationId()), stationId);
    }

    private String readFirstMatchingLine(Path file, String stationId) {
        try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
            return lines.filter(line -> line != null && !line.trim().isEmpty()).findFirst().orElse(null);
            return lines
                    .filter(line -> line != null && !line.trim().isEmpty())
                    .filter(line -> matchesRequestedStation(parseLogLine(line), stationId))
                    .findFirst()
                    .orElse(null);
        } catch (Exception e) {
            return null;
        }
    }

    private String readLastNonBlankLine(Path file) {
        try (RandomAccessFile randomAccessFile = new RandomAccessFile(file.toFile(), "r")) {
            long length = randomAccessFile.length();
            if (length <= 0) {
    private String readLastMatchingLine(Path file, String stationId) {
        try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
            List<String> matched = lines
                    .filter(line -> line != null && !line.trim().isEmpty())
                    .filter(line -> matchesRequestedStation(parseLogLine(line), stationId))
                    .collect(Collectors.toList());
            if (matched.isEmpty()) {
                return null;
            }
            ByteArrayOutputStream baos = new ByteArrayOutputStream();
            for (long pointer = length - 1; pointer >= 0; pointer--) {
                randomAccessFile.seek(pointer);
                int read = randomAccessFile.read();
                if (read == '\n' || read == '\r') {
                    if (baos.size() > 0) {
                        break;
                    }
                    continue;
                }
                baos.write(read);
            return matched.get(matched.size() - 1);
        } catch (Exception e) {
            return null;
        }
    }

    private DeviceDataLog parseLogLine(String line) {
        try {
            if (line == null || line.trim().isEmpty()) {
                return null;
            }
            byte[] bytes = baos.toByteArray();
            for (int i = 0, j = bytes.length - 1; i < j; i++, j--) {
                byte tmp = bytes[i];
                bytes[i] = bytes[j];
                bytes[j] = tmp;
            }
            String line = new String(bytes, StandardCharsets.UTF_8).trim();
            return line.isEmpty() ? null : line;
            return JSON.parseObject(line, DeviceDataLog.class);
        } catch (Exception e) {
            return null;
        }