fix: 同步 Jenkins 完成状态

src/deploy-runs/deploy-execution.service.spec.ts

@@ -301,6 +301,42 @@ describe('DeployExecutionService', () => {
     );
   });
 
+  it('recovers a completed Jenkins result from the console log when build status has no result yet', async () => {
+    const { audit, repository, service } = createService({
+      jenkinsHealth: { status: 'ok' },
+      notificationStatus: 'sent',
+      queueId: 'queue_log_success',
+      queueBuildNumber: 22,
+      logText: [
+        'deploy ok',
+        '[Pipeline] End of Pipeline',
+        'Finished: SUCCESS',
+      ].join('\n'),
+    });
+    const run = await service.execute(
+      await createRun(repository, 'run_log_success_001'),
+      project,
+    );
+
+    const result = await service.syncJenkinsRun(run, project);
+
+    expect(result.status).toBe('success');
+    expect(result.jenkinsBuildNumber).toBe(22);
+    expect(step(result, 'jenkins-build')?.status).toBe('success');
+    expect(step(result, 'health-check')?.status).toBe('success');
+    expect(step(result, 'notify-wecom')?.status).toBe('success');
+    expect(step(result, 'finish-release')?.status).toBe('success');
+    expect(audit.record).toHaveBeenCalledWith(
+      expect.objectContaining({
+        action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG',
+        after: expect.objectContaining({
+          buildNumber: 22,
+          result: 'SUCCESS',
+        }),
+      }),
+    );
+  });
+
   it('links a Jenkins build by queue id when the queue item has expired', async () => {
     const { audit, jenkins, repository, service } = createService({
       jenkinsHealth: { status: 'ok' },

src/deploy-runs/deploy-execution.service.ts

@@ -24,6 +24,8 @@ import {
 import { DeployRunRepository } from './deploy-run.repository';
 import { DeployRunStepStatus, DeployRunSummary } from './deploy-run.types';
 
+type JenkinsTerminalResult = NonNullable<JenkinsBuildStatus['result']>;
+
 @Injectable()
 export class DeployExecutionService {
   constructor(
@@ -206,6 +208,21 @@ export class DeployExecutionService {
     );
 
     if (buildStatus.building || !buildStatus.result) {
+      if (!buildStatus.building) {
+        const recoveredResult = await this.recoverCompletedBuildResultFromLog(
+          currentRun,
+          environment.jenkinsJobPath,
+        );
+
+        if (recoveredResult) {
+          return this.completeJenkinsRunWithResult(
+            currentRun,
+            recoveredResult,
+            environment.jenkinsJobPath,
+          );
+        }
+      }
+
       return this.markStep(
         currentRun,
         'jenkins-build',
@@ -214,17 +231,44 @@ export class DeployExecutionService {
       );
     }
 
-    if (buildStatus.result === 'SUCCESS') {
+    return this.completeJenkinsRunWithResult(
-      return this.completeSuccessfulJenkinsRun(currentRun);
-    }
-
-    return this.failJenkinsRun(
       currentRun,
       buildStatus.result,
       environment.jenkinsJobPath,
     );
   }
 
+  async syncJenkinsRunFromCompletedLog(
+    run: DeployRunSummary,
+    project: ProjectSummary,
+    logText: string,
+    hasMore: boolean,
+  ): Promise<DeployRunSummary> {
+    if (['success', 'failed', 'canceled'].includes(run.status) || hasMore) {
+      return run;
+    }
+
+    const result = this.extractJenkinsFinishedResult(logText);
+
+    if (!result) {
+      return run;
+    }
+
+    const environment = this.findEnvironment(project, run);
+    await this.recordJenkinsLogResultRecovered(
+      run,
+      result,
+      environment.jenkinsJobPath,
+    );
+
+    return this.completeJenkinsRunWithResult(
+      run,
+      result,
+      environment.jenkinsJobPath,
+      logText,
+    );
+  }
+
   private async resolveJenkinsBuildReference(
     run: DeployRunSummary,
     environment: ProjectEnvironmentSummary,
@@ -673,10 +717,13 @@ export class DeployExecutionService {
 
   private async failJenkinsRun(
     run: DeployRunSummary,
-    result: Exclude<NonNullable<JenkinsBuildStatus['result']>, 'SUCCESS'>,
+    result: Exclude<JenkinsTerminalResult, 'SUCCESS'>,
     jobPath: string,
+    logText?: string,
   ): Promise<DeployRunSummary> {
-    const logExcerpt = await this.readFailureLogExcerpt(run, jobPath);
+    const logExcerpt = logText
+      ? this.toLogExcerpt(logText)
+      : await this.readFailureLogExcerpt(run, jobPath);
     const failureSummary = `Jenkins 构建 #${run.jenkinsBuildNumber ?? '-'} ${result}`;
     let currentRun =
       (await this.deployRunRepository.updateStep(
@@ -729,19 +776,106 @@ export class DeployExecutionService {
         jobPath,
         run.jenkinsBuildNumber,
       );
-      const logExcerpt = log.text
+      return this.toLogExcerpt(log.text);
-        .split(/\r?\n/)
-        .filter(Boolean)
-        .slice(-24)
-        .join('\n')
-        .slice(-4000);
-
-      return redactSensitive({ logExcerpt }).logExcerpt;
     } catch {
       return undefined;
     }
   }
 
+  private async recoverCompletedBuildResultFromLog(
+    run: DeployRunSummary,
+    jobPath: string,
+  ): Promise<JenkinsTerminalResult | undefined> {
+    if (!run.jenkinsBuildNumber) {
+      return undefined;
+    }
+
+    try {
+      const log = await this.jenkinsClient.getProgressiveText(
+        jobPath,
+        run.jenkinsBuildNumber,
+      );
+
+      if (log.hasMore) {
+        return undefined;
+      }
+
+      const result = this.extractJenkinsFinishedResult(log.text);
+
+      if (!result) {
+        return undefined;
+      }
+
+      await this.recordJenkinsLogResultRecovered(run, result, jobPath);
+
+      return result;
+    } catch {
+      return undefined;
+    }
+  }
+
+  private async recordJenkinsLogResultRecovered(
+    run: DeployRunSummary,
+    result: JenkinsTerminalResult,
+    jobPath: string,
+  ): Promise<void> {
+    await this.auditService.record({
+      action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG',
+      resourceType: 'deploy_run',
+      resourceId: run.id,
+      actorName: run.operator,
+      after: {
+        jobPath,
+        buildNumber: run.jenkinsBuildNumber,
+        result,
+      },
+    });
+  }
+
+  private completeJenkinsRunWithResult(
+    run: DeployRunSummary,
+    result: JenkinsTerminalResult,
+    jobPath: string,
+    logText?: string,
+  ): Promise<DeployRunSummary> {
+    if (result === 'SUCCESS') {
+      return this.completeSuccessfulJenkinsRun(run);
+    }
+
+    return this.failJenkinsRun(run, result, jobPath, logText);
+  }
+
+  private extractJenkinsFinishedResult(
+    logText: string,
+  ): JenkinsTerminalResult | undefined {
+    const match = logText.match(
+      /(?:^|\r?\n)\s*Finished:\s*(SUCCESS|FAILURE|ABORTED|UNSTABLE)\b/i,
+    );
+    const result = match?.[1]?.toUpperCase();
+
+    if (
+      result === 'SUCCESS' ||
+      result === 'FAILURE' ||
+      result === 'ABORTED' ||
+      result === 'UNSTABLE'
+    ) {
+      return result;
+    }
+
+    return undefined;
+  }
+
+  private toLogExcerpt(logText: string): string {
+    const logExcerpt = logText
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .slice(-24)
+      .join('\n')
+      .slice(-4000);
+
+    return redactSensitive({ logExcerpt }).logExcerpt;
+  }
+
   private findEnvironment(
     project: ProjectSummary,
     run: DeployRunSummary,

src/deploy-runs/deploy-runs.service.spec.ts

@@ -114,13 +114,22 @@ describe('DeployRunsService', () => {
       ),
     } satisfies Pick<DeployNotificationService, 'sendDeployRunEvent'>;
     const execution = {
-      execute: jest.fn((run: DeployRunSummary) =>
+      execute: jest.fn(
-        Promise.resolve({ ...run, status: 'success' as const }),
+        (run: DeployRunSummary): Promise<DeployRunSummary> =>
+          Promise.resolve({ ...run, status: 'success' as const }),
       ),
-      syncJenkinsRun: jest.fn((run: DeployRunSummary) =>
+      syncJenkinsRun: jest.fn(
-        Promise.resolve({ ...run, status: 'success' as const }),
+        (run: DeployRunSummary): Promise<DeployRunSummary> =>
+          Promise.resolve({ ...run, status: 'success' as const }),
       ),
-    } satisfies Pick<DeployExecutionService, 'execute' | 'syncJenkinsRun'>;
+      syncJenkinsRunFromCompletedLog: jest.fn(
+        (run: DeployRunSummary): Promise<DeployRunSummary> =>
+          Promise.resolve(run),
+      ),
+    } satisfies Pick<
+      DeployExecutionService,
+      'execute' | 'syncJenkinsRun' | 'syncJenkinsRunFromCompletedLog'
+    >;
     const jenkins = {
       healthSummary: jest.fn(() => ({ status: 'ok' as const })),
       cancelQueueItem: jest.fn(() =>
@@ -137,20 +146,14 @@ describe('DeployRunsService', () => {
           requested: true as const,
         }),
       ),
-      findBuildByQueueId: jest.fn(
+      findBuildByQueueId: jest.fn((_jobPath: string, _queueId: string) => {
-        (_jobPath: string, _queueId: string) => {
+        void _jobPath;
-          void _jobPath;
+        void _queueId;
-          void _queueId;
 
-          return Promise.resolve(undefined);
+        return Promise.resolve(undefined);
-        },
+      }),
-      ),
       findLatestBuildByParameter: jest.fn(
-        (
+        (_jobPath: string, _parameterName: string, _parameterValue: string) => {
-          _jobPath: string,
-          _parameterName: string,
-          _parameterValue: string,
-        ) => {
           void _jobPath;
           void _parameterName;
           void _parameterValue;
@@ -188,6 +191,23 @@ describe('DeployRunsService', () => {
       AuthService,
       'listAllowedProjectKeys' | 'requireProjectPermission' | 'assertSuperAdmin'
     >;
+    const service = new DeployRunsService(
+      repository as unknown as DeployRunRepository,
+      projects as unknown as ProjectsService,
+      audit as unknown as AuditService,
+      notification as unknown as DeployNotificationService,
+      execution as unknown as DeployExecutionService,
+      jenkins as unknown as JenkinsClient,
+      auth as unknown as AuthService,
+    );
+    const waitBeforeRetryingJenkinsLogBuildLink = jest
+      .spyOn(
+        service as unknown as {
+          waitBeforeRetryingJenkinsLogBuildLink: () => Promise<void>;
+        },
+        'waitBeforeRetryingJenkinsLogBuildLink',
+      )
+      .mockResolvedValue(undefined);
 
     return {
       audit,
@@ -197,15 +217,8 @@ describe('DeployRunsService', () => {
       notification,
       projects,
       repository,
-      service: new DeployRunsService(
+      service,
-        repository as unknown as DeployRunRepository,
+      waitBeforeRetryingJenkinsLogBuildLink,
-        projects as unknown as ProjectsService,
-        audit as unknown as AuditService,
-        notification as unknown as DeployNotificationService,
-        execution as unknown as DeployExecutionService,
-        jenkins as unknown as JenkinsClient,
-        auth as unknown as AuthService,
-      ),
     };
   }
 
@@ -432,6 +445,48 @@ describe('DeployRunsService', () => {
     expect(run).toBe(runningRun);
   });
 
+  it('retries read-through Jenkins sync after a run version conflict', async () => {
+    const runningRun: DeployRunSummary = {
+      ...baseRun,
+      id: 'run_running',
+      status: 'running',
+      jenkinsBuildNumber: 7,
+    };
+    const latestRun: DeployRunSummary = {
+      ...runningRun,
+      version: 2,
+    };
+    const { execution, repository, service } = createService({
+      runs: [runningRun],
+    });
+    repository.findById
+      .mockResolvedValueOnce(runningRun)
+      .mockResolvedValueOnce(latestRun);
+    execution.syncJenkinsRun
+      .mockRejectedValueOnce(
+        new AppError(
+          'CONFLICT',
+          '发布记录 run_running 已被其他流程更新，请刷新后重试',
+          409,
+        ),
+      )
+      .mockResolvedValueOnce({ ...latestRun, status: 'success' });
+
+    const run = await service.getRun('run_running');
+
+    expect(run.status).toBe('success');
+    expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith(
+      1,
+      runningRun,
+      expect.objectContaining({ key: runningRun.projectKey }),
+    );
+    expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith(
+      2,
+      latestRun,
+      expect.objectContaining({ key: runningRun.projectKey }),
+    );
+  });
+
   it('links a missing Jenkins build before reading logs', async () => {
     const unlinkedRun: DeployRunSummary = {
       ...baseRun,
@@ -484,6 +539,38 @@ describe('DeployRunsService', () => {
     );
   });
 
+  it('uses completed Jenkins logs to advance status while reading logs', async () => {
+    const runningRun: DeployRunSummary = {
+      ...baseRun,
+      id: 'run_log_finished',
+      status: 'running',
+      jenkinsBuildNumber: 8,
+    };
+    const { execution, jenkins, service } = createService({
+      runs: [runningRun],
+    });
+    execution.syncJenkinsRun.mockResolvedValueOnce(runningRun);
+    jenkins.getProgressiveText.mockResolvedValueOnce({
+      text: [
+        'deploy ok',
+        '[Pipeline] End of Pipeline',
+        'Finished: SUCCESS',
+      ].join('\n'),
+      nextStart: 56,
+      hasMore: false,
+    });
+
+    const log = await service.getJenkinsLog('run_log_finished', undefined);
+
+    expect(log.text).toContain('Finished: SUCCESS');
+    expect(execution.syncJenkinsRunFromCompletedLog).toHaveBeenCalledWith(
+      runningRun,
+      expect.objectContaining({ key: 'access-manage' }),
+      expect.stringContaining('Finished: SUCCESS'),
+      false,
+    );
+  });
+
   it('links a missing Jenkins build by queue id before reading logs', async () => {
     const unlinkedRun: DeployRunSummary = {
       ...baseRun,
@@ -535,6 +622,47 @@ describe('DeployRunsService', () => {
     );
   });
 
+  it('waits until Jenkins has created a build before reading logs', async () => {
+    const unlinkedRun: DeployRunSummary = {
+      ...baseRun,
+      id: 'run_log_wait_queue_build',
+      status: 'running',
+      jenkinsQueueId: '439',
+    };
+    const { jenkins, service, waitBeforeRetryingJenkinsLogBuildLink } =
+      createService({
+        runs: [unlinkedRun],
+      });
+    const findBuildByQueueId =
+      jenkins.findBuildByQueueId as unknown as jest.MockedFunction<
+        JenkinsClient['findBuildByQueueId']
+      >;
+    findBuildByQueueId.mockResolvedValueOnce(undefined).mockResolvedValueOnce({
+      jobPath: 'my-project-prod/prod-access-manage-tag',
+      buildNumber: 8,
+      building: true,
+      url: 'https://jenkins.example/job/8/',
+      queueId: '439',
+    });
+
+    const log = await service.getJenkinsLog(
+      'run_log_wait_queue_build',
+      undefined,
+    );
+
+    expect(log.buildNumber).toBe(8);
+    expect(findBuildByQueueId).toHaveBeenCalledTimes(2);
+    expect(waitBeforeRetryingJenkinsLogBuildLink).toHaveBeenCalledTimes(1);
+    expect(jenkins.getProgressiveText).toHaveBeenCalledWith(
+      'my-project-dev/test-access-manage-develop',
+      8,
+      0,
+    );
+    expect(
+      waitBeforeRetryingJenkinsLogBuildLink.mock.invocationCallOrder[0],
+    ).toBeLessThan(jenkins.getProgressiveText.mock.invocationCallOrder[0]);
+  });
+
   it('cancels a queued Jenkins execution before marking the run canceled', async () => {
     const queuedRun: DeployRunSummary = {
       ...baseRun,
@@ -625,9 +753,7 @@ describe('DeployRunsService', () => {
   it('rejects non-terminal retry requests before creating a new run', async () => {
     const { repository, service } = createService();
 
-    await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf(
+    await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf(AppError);
-      AppError,
-    );
     expect(repository.create).not.toHaveBeenCalled();
   });
 

src/deploy-runs/deploy-runs.service.ts

@@ -31,6 +31,9 @@ type ReadRunsOptions = {
   syncJenkins?: boolean;
 };
 
+const jenkinsSyncConflictRetryCount = 2;
+const jenkinsLogBuildLinkMaxAttempts = 5;
+const jenkinsLogBuildLinkRetryDelayMs = 1000;
 const escapeControl = String.fromCharCode(27);
 const jenkinsConsoleNotePattern = new RegExp(
   `${escapeControl}\\[8mha:/{4}[A-Za-z0-9+/=]+${escapeControl}\\[0m`,
@@ -274,7 +277,7 @@ export class DeployRunsService {
 
     const project = await this.projectsService.getProject(run.projectKey);
 
-    return this.deployExecutionService.syncJenkinsRun(run, project);
+    return this.syncJenkinsRunWithRetry(run, project);
   }
 
   async getJenkinsLog(
@@ -288,7 +291,7 @@ export class DeployRunsService {
       run.jenkinsBuildNumber === undefined ||
       run.jenkinsBuildNumber === null
     ) {
-      run = await this.linkMissingJenkinsBuildForLog(run);
+      run = await this.waitForJenkinsBuildForLog(run);
     }
 
     if (
@@ -306,17 +309,24 @@ export class DeployRunsService {
       );
     }
 
+    const jenkinsBuildNumber = run.jenkinsBuildNumber;
     const project = await this.projectsService.getProject(run.projectKey);
     const environment = this.findEnvironment(project, run);
     const normalizedStart = this.normalizeLogStart(start);
     const log = await this.jenkinsClient.getProgressiveText(
       environment.jenkinsJobPath,
-      run.jenkinsBuildNumber,
+      jenkinsBuildNumber,
       normalizedStart,
     );
     const text = redactSensitive({
       text: this.cleanJenkinsLogText(log.text),
     }).text;
+    run = await this.syncJenkinsRunFromLogForRead(
+      run,
+      project,
+      text,
+      log.hasMore,
+    );
 
     await this.auditService.record({
       action: 'JENKINS_LOG_VIEWED',
@@ -326,7 +336,7 @@ export class DeployRunsService {
       after: {
         projectKey: run.projectKey,
         environment: run.environment,
-        jenkinsBuildNumber: run.jenkinsBuildNumber,
+        jenkinsBuildNumber,
         start: normalizedStart,
         nextStart: log.nextStart,
         hasMore: log.hasMore,
@@ -337,7 +347,7 @@ export class DeployRunsService {
     return {
       runId: run.id,
       jobPath: environment.jenkinsJobPath,
-      buildNumber: run.jenkinsBuildNumber,
+      buildNumber: jenkinsBuildNumber,
       start: normalizedStart,
       nextStart: log.nextStart,
       hasMore: log.hasMore,
@@ -361,10 +371,7 @@ export class DeployRunsService {
     for (const run of candidates) {
       try {
         const project = await this.projectsService.getProject(run.projectKey);
-        const syncedRun = await this.deployExecutionService.syncJenkinsRun(
+        const syncedRun = await this.syncJenkinsRunWithRetry(run, project);
-          run,
-          project,
-        );
         syncedRuns.push(syncedRun);
       } catch (error) {
         failedRuns.push({
@@ -443,7 +450,7 @@ export class DeployRunsService {
     try {
       const project = await this.projectsService.getProject(run.projectKey);
 
-      return await this.deployExecutionService.syncJenkinsRun(run, project);
+      return await this.syncJenkinsRunWithRetry(run, project);
     } catch {
       return run;
     }
@@ -480,7 +487,9 @@ export class DeployRunsService {
                 ? 'running'
                 : run.status,
             jenkinsQueueId:
-              'queueId' in matchedBuild ? matchedBuild.queueId : run.jenkinsQueueId,
+              'queueId' in matchedBuild
+                ? matchedBuild.queueId
+                : run.jenkinsQueueId,
             jenkinsBuildNumber: matchedBuild.buildNumber,
             jenkinsBuildUrl: matchedBuild.url,
           },
@@ -508,15 +517,133 @@ export class DeployRunsService {
         },
       });
 
-      return await this.deployExecutionService.syncJenkinsRun(
+      return await this.syncJenkinsRunWithRetry(linkedRun, project);
-        linkedRun,
+    } catch {
+      return run;
+    }
+  }
+
+  private async waitForJenkinsBuildForLog(
+    run: DeployRunSummary,
+  ): Promise<DeployRunSummary> {
+    let currentRun = run;
+
+    for (
+      let attempt = 0;
+      attempt < jenkinsLogBuildLinkMaxAttempts;
+      attempt += 1
+    ) {
+      currentRun = await this.linkMissingJenkinsBuildForLog(currentRun);
+
+      if (
+        currentRun.jenkinsBuildNumber !== undefined &&
+        currentRun.jenkinsBuildNumber !== null
+      ) {
+        return currentRun;
+      }
+
+      if (attempt < jenkinsLogBuildLinkMaxAttempts - 1) {
+        await this.waitBeforeRetryingJenkinsLogBuildLink();
+      }
+    }
+
+    return currentRun;
+  }
+
+  private waitBeforeRetryingJenkinsLogBuildLink(): Promise<void> {
+    return new Promise((resolve) => {
+      const timer = setTimeout(resolve, jenkinsLogBuildLinkRetryDelayMs);
+      timer.unref?.();
+    });
+  }
+
+  private async syncJenkinsRunFromLogForRead(
+    run: DeployRunSummary,
+    project: ProjectSummary,
+    logText: string,
+    hasMore: boolean,
+  ): Promise<DeployRunSummary> {
+    try {
+      return await this.syncJenkinsLogResultWithRetry(
+        run,
         project,
+        logText,
+        hasMore,
       );
     } catch {
       return run;
     }
   }
 
+  private async syncJenkinsRunWithRetry(
+    run: DeployRunSummary,
+    project: ProjectSummary,
+  ): Promise<DeployRunSummary> {
+    return this.retryJenkinsSync(run, (currentRun) =>
+      this.deployExecutionService.syncJenkinsRun(currentRun, project),
+    );
+  }
+
+  private async syncJenkinsLogResultWithRetry(
+    run: DeployRunSummary,
+    project: ProjectSummary,
+    logText: string,
+    hasMore: boolean,
+  ): Promise<DeployRunSummary> {
+    return this.retryJenkinsSync(run, (currentRun) =>
+      this.deployExecutionService.syncJenkinsRunFromCompletedLog(
+        currentRun,
+        project,
+        logText,
+        hasMore,
+      ),
+    );
+  }
+
+  private async retryJenkinsSync(
+    run: DeployRunSummary,
+    operation: (currentRun: DeployRunSummary) => Promise<DeployRunSummary>,
+  ): Promise<DeployRunSummary> {
+    let currentRun = run;
+
+    for (
+      let attempt = 0;
+      attempt <= jenkinsSyncConflictRetryCount;
+      attempt += 1
+    ) {
+      try {
+        return await operation(currentRun);
+      } catch (error) {
+        if (
+          !this.isRunVersionConflict(error) ||
+          attempt === jenkinsSyncConflictRetryCount
+        ) {
+          throw error;
+        }
+
+        const latestRun = await this.deployRunRepository.findById(
+          currentRun.id,
+        );
+
+        if (!latestRun) {
+          throw error;
+        }
+
+        if (!this.isJenkinsSyncCandidate(latestRun)) {
+          return latestRun;
+        }
+
+        currentRun = latestRun;
+      }
+    }
+
+    return currentRun;
+  }
+
+  private isRunVersionConflict(error: unknown): boolean {
+    return error instanceof AppError && error.code === 'CONFLICT';
+  }
+
   private normalizeLogStart(start: string | undefined): number {
     if (start === undefined || start === '') {
       return 0;