From 892e45141a55bfea0e28c6a34a384f2ce8ee7d16 Mon Sep 17 00:00:00 2001
From: Junjie <fallin.jie@qq.com>
Date: 星期一, 27 四月 2026 13:20:17 +0800
Subject: [PATCH] fix: make auto tune rollback auditable atomically
---
src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java | 138 ++++++++++++++++++++++++++++++++++++++--------
1 files changed, 114 insertions(+), 24 deletions(-)
diff --git a/src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java b/src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java
index a8a95e5..3c42210 100644
--- a/src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java
+++ b/src/main/java/com/zy/ai/service/impl/AutoTuneApplyServiceImpl.java
@@ -197,30 +197,58 @@
public AutoTuneApplyResult rollbackLastSuccessfulJob(String reason) {
Date now = new Date();
AiAutoTuneJob rollbackJob = createRollbackJob(reason, now);
- aiAutoTuneJobService.save(rollbackJob);
List<AiAutoTuneChange> sourceChanges = findLatestSuccessfulChanges();
if (sourceChanges.isEmpty()) {
- rollbackJob.setStatus(AutoTuneJobStatus.REJECTED.getCode());
- rollbackJob.setFinishTime(now);
- rollbackJob.setRejectCount(0);
- rollbackJob.setSuccessCount(0);
- rollbackJob.setSummary("鏈壘鍒板彲鍥炴粴鐨勬垚鍔熻皟鍙傝褰�");
- rollbackJob.setErrorMessage("鏈壘鍒板彲鍥炴粴鐨勬垚鍔熻皟鍙傝褰�");
- aiAutoTuneJobService.updateById(rollbackJob);
+ persistNoRollbackSourceJobInTransaction(rollbackJob, now);
return buildResult(rollbackJob, new ArrayList<>(), false);
}
- List<AiAutoTuneChange> rollbackChanges = new ArrayList<>();
- try {
- rollbackChanges = rollbackChangesInTransaction(rollbackJob.getId(), sourceChanges, now);
- } catch (RuntimeException exception) {
- rollbackChanges = buildFailedRollbackChanges(rollbackJob.getId(), sourceChanges, exception, now);
+ String lockKey = RedisKeyType.AI_AUTO_TUNE_APPLY_LOCK.key;
+ String lockToken = UUID.randomUUID().toString();
+ if (!redisUtil.trySetStringIfAbsent(lockKey, lockToken, APPLY_LOCK_SECONDS)) {
+ return rejectRollbackForUnavailableLock(reason, sourceChanges, now, lockKey);
}
- aiAutoTuneChangeService.saveBatch(rollbackChanges);
- finishRollbackJob(rollbackJob, rollbackChanges, now);
- aiAutoTuneJobService.updateById(rollbackJob);
- return buildResult(rollbackJob, rollbackChanges, false);
+
+ try {
+ try {
+ RollbackPersistenceResult persistenceResult = persistRollbackResultInTransaction(
+ rollbackJob,
+ sourceChanges,
+ now
+ );
+ refreshRollbackConfigCacheIfNeeded(persistenceResult);
+ return buildResult(rollbackJob, persistenceResult.getRollbackChanges(), false);
+ } catch (RuntimeException exception) {
+ Date failureNow = new Date();
+ AiAutoTuneJob failureJob = createRollbackJob(reason, failureNow);
+ RollbackPersistenceResult persistenceResult = persistFailedRollbackResultInTransaction(
+ failureJob,
+ sourceChanges,
+ exception,
+ failureNow
+ );
+ return buildResult(failureJob, persistenceResult.getRollbackChanges(), false);
+ }
+ } finally {
+ redisUtil.compareAndDelete(lockKey, lockToken);
+ }
+ }
+
+ private AutoTuneApplyResult rejectRollbackForUnavailableLock(String reason,
+ List<AiAutoTuneChange> sourceChanges,
+ Date now,
+ String lockKey) {
+ boolean lockKeyExists = redisUtil.hasKey(lockKey);
+ LOGGER.warn("鐢宠AI鑷姩璋冨弬 rollback 閿佸け璐ワ紝lockKey={}, lockKeyExists={}", lockKey, lockKeyExists);
+ AiAutoTuneJob rollbackJob = createRollbackJob(reason, now);
+ RollbackPersistenceResult persistenceResult = persistFailedRollbackResultInTransaction(
+ rollbackJob,
+ sourceChanges,
+ new IllegalStateException(APPLY_LOCK_BUSY_REASON),
+ now
+ );
+ return buildResult(rollbackJob, persistenceResult.getRollbackChanges(), false);
}
private List<ValidatedChange> validateChanges(AutoTuneApplyRequest request, boolean dryRun, Date now) {
@@ -466,14 +494,55 @@
}
}
- private List<AiAutoTuneChange> rollbackChangesInTransaction(Long rollbackJobId,
- List<AiAutoTuneChange> sourceChanges,
- Date now) {
+ private void persistNoRollbackSourceJobInTransaction(AiAutoTuneJob rollbackJob, Date now) {
TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
- return transactionTemplate.execute(status -> rollbackChanges(rollbackJobId, sourceChanges, now));
+ transactionTemplate.executeWithoutResult(status -> {
+ saveJob(rollbackJob);
+ rollbackJob.setStatus(AutoTuneJobStatus.REJECTED.getCode());
+ rollbackJob.setFinishTime(now);
+ rollbackJob.setRejectCount(0);
+ rollbackJob.setSuccessCount(0);
+ rollbackJob.setSummary("鏈壘鍒板彲鍥炴粴鐨勬垚鍔熻皟鍙傝褰�");
+ rollbackJob.setErrorMessage("鏈壘鍒板彲鍥炴粴鐨勬垚鍔熻皟鍙傝褰�");
+ updateJob(rollbackJob);
+ });
}
- private List<AiAutoTuneChange> rollbackChanges(Long rollbackJobId, List<AiAutoTuneChange> sourceChanges, Date now) {
+ private RollbackPersistenceResult persistRollbackResultInTransaction(AiAutoTuneJob rollbackJob,
+ List<AiAutoTuneChange> sourceChanges,
+ Date now) {
+ TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
+ return transactionTemplate.execute(status -> {
+ saveJob(rollbackJob);
+ RollbackPersistenceResult rollbackResult = rollbackChanges(rollbackJob.getId(), sourceChanges, now);
+ saveAuditChanges(rollbackResult.getRollbackChanges());
+ finishRollbackJob(rollbackJob, rollbackResult.getRollbackChanges(), now);
+ updateJob(rollbackJob);
+ return rollbackResult;
+ });
+ }
+
+ private RollbackPersistenceResult persistFailedRollbackResultInTransaction(AiAutoTuneJob rollbackJob,
+ List<AiAutoTuneChange> sourceChanges,
+ RuntimeException exception,
+ Date now) {
+ TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
+ return transactionTemplate.execute(status -> {
+ saveJob(rollbackJob);
+ List<AiAutoTuneChange> rollbackChanges = buildFailedRollbackChanges(
+ rollbackJob.getId(),
+ sourceChanges,
+ exception,
+ now
+ );
+ saveAuditChanges(rollbackChanges);
+ finishRollbackJob(rollbackJob, rollbackChanges, now);
+ updateJob(rollbackJob);
+ return new RollbackPersistenceResult(rollbackChanges, false);
+ });
+ }
+
+ private RollbackPersistenceResult rollbackChanges(Long rollbackJobId, List<AiAutoTuneChange> sourceChanges, Date now) {
List<AiAutoTuneChange> rollbackChanges = new ArrayList<>();
boolean refreshConfigCache = false;
for (AiAutoTuneChange sourceChange : sourceChanges) {
@@ -492,10 +561,13 @@
rollbackChange.setResultStatus(ChangeStatus.SUCCESS.getCode());
rollbackChanges.add(rollbackChange);
}
- if (refreshConfigCache) {
+ return new RollbackPersistenceResult(rollbackChanges, refreshConfigCache);
+ }
+
+ private void refreshRollbackConfigCacheIfNeeded(RollbackPersistenceResult persistenceResult) {
+ if (persistenceResult != null && persistenceResult.isRefreshConfigCache()) {
configService.refreshSystemConfigCache();
}
- return rollbackChanges;
}
private void writeValue(String targetType, String targetId, String targetKey, String value) {
@@ -936,6 +1008,24 @@
}
}
+ private static class RollbackPersistenceResult {
+ private final List<AiAutoTuneChange> rollbackChanges;
+ private final boolean refreshConfigCache;
+
+ private RollbackPersistenceResult(List<AiAutoTuneChange> rollbackChanges, boolean refreshConfigCache) {
+ this.rollbackChanges = rollbackChanges == null ? new ArrayList<>() : rollbackChanges;
+ this.refreshConfigCache = refreshConfigCache;
+ }
+
+ public List<AiAutoTuneChange> getRollbackChanges() {
+ return rollbackChanges;
+ }
+
+ public boolean isRefreshConfigCache() {
+ return refreshConfigCache;
+ }
+ }
+
private static class ValidatedChange {
private final AutoTuneChangeCommand command;
private final String targetType;
--
Gitblit v1.9.1