From 1b645dd9c8bd84f697b724712800fafa468d453d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B9=9B=E5=85=AE?= Date: Fri, 12 Jun 2026 14:43:46 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=90=8C=E6=AD=A5=20Jenkins=20=E5=AE=8C?= =?UTF-8?q?=E6=88=90=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../deploy-execution.service.spec.ts | 36 ++++ src/deploy-runs/deploy-execution.service.ts | 164 ++++++++++++++-- src/deploy-runs/deploy-runs.service.spec.ts | 184 +++++++++++++++--- src/deploy-runs/deploy-runs.service.ts | 153 +++++++++++++-- 4 files changed, 480 insertions(+), 57 deletions(-) diff --git a/src/deploy-runs/deploy-execution.service.spec.ts b/src/deploy-runs/deploy-execution.service.spec.ts index a286cf7..20efab6 100644 --- a/src/deploy-runs/deploy-execution.service.spec.ts +++ b/src/deploy-runs/deploy-execution.service.spec.ts @@ -301,6 +301,42 @@ describe('DeployExecutionService', () => { ); }); + it('recovers a completed Jenkins result from the console log when build status has no result yet', async () => { + const { audit, repository, service } = createService({ + jenkinsHealth: { status: 'ok' }, + notificationStatus: 'sent', + queueId: 'queue_log_success', + queueBuildNumber: 22, + logText: [ + 'deploy ok', + '[Pipeline] End of Pipeline', + 'Finished: SUCCESS', + ].join('\n'), + }); + const run = await service.execute( + await createRun(repository, 'run_log_success_001'), + project, + ); + + const result = await service.syncJenkinsRun(run, project); + + expect(result.status).toBe('success'); + expect(result.jenkinsBuildNumber).toBe(22); + expect(step(result, 'jenkins-build')?.status).toBe('success'); + expect(step(result, 'health-check')?.status).toBe('success'); + expect(step(result, 'notify-wecom')?.status).toBe('success'); + expect(step(result, 'finish-release')?.status).toBe('success'); + expect(audit.record).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG', + after: expect.objectContaining({ + buildNumber: 22, + result: 'SUCCESS', + }), + }), + ); + }); + it('links a Jenkins build by queue id when the queue item has expired', async () => { const { audit, jenkins, repository, service } = createService({ jenkinsHealth: { status: 'ok' }, diff --git a/src/deploy-runs/deploy-execution.service.ts b/src/deploy-runs/deploy-execution.service.ts index 7a50c22..4fa4ddd 100644 --- a/src/deploy-runs/deploy-execution.service.ts +++ b/src/deploy-runs/deploy-execution.service.ts @@ -24,6 +24,8 @@ import { import { DeployRunRepository } from './deploy-run.repository'; import { DeployRunStepStatus, DeployRunSummary } from './deploy-run.types'; +type JenkinsTerminalResult = NonNullable; + @Injectable() export class DeployExecutionService { constructor( @@ -206,6 +208,21 @@ export class DeployExecutionService { ); if (buildStatus.building || !buildStatus.result) { + if (!buildStatus.building) { + const recoveredResult = await this.recoverCompletedBuildResultFromLog( + currentRun, + environment.jenkinsJobPath, + ); + + if (recoveredResult) { + return this.completeJenkinsRunWithResult( + currentRun, + recoveredResult, + environment.jenkinsJobPath, + ); + } + } + return this.markStep( currentRun, 'jenkins-build', @@ -214,17 +231,44 @@ export class DeployExecutionService { ); } - if (buildStatus.result === 'SUCCESS') { - return this.completeSuccessfulJenkinsRun(currentRun); - } - - return this.failJenkinsRun( + return this.completeJenkinsRunWithResult( currentRun, buildStatus.result, environment.jenkinsJobPath, ); } + async syncJenkinsRunFromCompletedLog( + run: DeployRunSummary, + project: ProjectSummary, + logText: string, + hasMore: boolean, + ): Promise { + if (['success', 'failed', 'canceled'].includes(run.status) || hasMore) { + return run; + } + + const result = this.extractJenkinsFinishedResult(logText); + + if (!result) { + return run; + } + + const environment = this.findEnvironment(project, run); + await this.recordJenkinsLogResultRecovered( + run, + result, + environment.jenkinsJobPath, + ); + + return this.completeJenkinsRunWithResult( + run, + result, + environment.jenkinsJobPath, + logText, + ); + } + private async resolveJenkinsBuildReference( run: DeployRunSummary, environment: ProjectEnvironmentSummary, @@ -673,10 +717,13 @@ export class DeployExecutionService { private async failJenkinsRun( run: DeployRunSummary, - result: Exclude, 'SUCCESS'>, + result: Exclude, jobPath: string, + logText?: string, ): Promise { - const logExcerpt = await this.readFailureLogExcerpt(run, jobPath); + const logExcerpt = logText + ? this.toLogExcerpt(logText) + : await this.readFailureLogExcerpt(run, jobPath); const failureSummary = `Jenkins 构建 #${run.jenkinsBuildNumber ?? '-'} ${result}`; let currentRun = (await this.deployRunRepository.updateStep( @@ -729,19 +776,106 @@ export class DeployExecutionService { jobPath, run.jenkinsBuildNumber, ); - const logExcerpt = log.text - .split(/\r?\n/) - .filter(Boolean) - .slice(-24) - .join('\n') - .slice(-4000); - - return redactSensitive({ logExcerpt }).logExcerpt; + return this.toLogExcerpt(log.text); } catch { return undefined; } } + private async recoverCompletedBuildResultFromLog( + run: DeployRunSummary, + jobPath: string, + ): Promise { + if (!run.jenkinsBuildNumber) { + return undefined; + } + + try { + const log = await this.jenkinsClient.getProgressiveText( + jobPath, + run.jenkinsBuildNumber, + ); + + if (log.hasMore) { + return undefined; + } + + const result = this.extractJenkinsFinishedResult(log.text); + + if (!result) { + return undefined; + } + + await this.recordJenkinsLogResultRecovered(run, result, jobPath); + + return result; + } catch { + return undefined; + } + } + + private async recordJenkinsLogResultRecovered( + run: DeployRunSummary, + result: JenkinsTerminalResult, + jobPath: string, + ): Promise { + await this.auditService.record({ + action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG', + resourceType: 'deploy_run', + resourceId: run.id, + actorName: run.operator, + after: { + jobPath, + buildNumber: run.jenkinsBuildNumber, + result, + }, + }); + } + + private completeJenkinsRunWithResult( + run: DeployRunSummary, + result: JenkinsTerminalResult, + jobPath: string, + logText?: string, + ): Promise { + if (result === 'SUCCESS') { + return this.completeSuccessfulJenkinsRun(run); + } + + return this.failJenkinsRun(run, result, jobPath, logText); + } + + private extractJenkinsFinishedResult( + logText: string, + ): JenkinsTerminalResult | undefined { + const match = logText.match( + /(?:^|\r?\n)\s*Finished:\s*(SUCCESS|FAILURE|ABORTED|UNSTABLE)\b/i, + ); + const result = match?.[1]?.toUpperCase(); + + if ( + result === 'SUCCESS' || + result === 'FAILURE' || + result === 'ABORTED' || + result === 'UNSTABLE' + ) { + return result; + } + + return undefined; + } + + private toLogExcerpt(logText: string): string { + const logExcerpt = logText + .split(/\r?\n/) + .filter(Boolean) + .slice(-24) + .join('\n') + .slice(-4000); + + return redactSensitive({ logExcerpt }).logExcerpt; + } + private findEnvironment( project: ProjectSummary, run: DeployRunSummary, diff --git a/src/deploy-runs/deploy-runs.service.spec.ts b/src/deploy-runs/deploy-runs.service.spec.ts index 43f3cf1..08464ed 100644 --- a/src/deploy-runs/deploy-runs.service.spec.ts +++ b/src/deploy-runs/deploy-runs.service.spec.ts @@ -114,13 +114,22 @@ describe('DeployRunsService', () => { ), } satisfies Pick; const execution = { - execute: jest.fn((run: DeployRunSummary) => - Promise.resolve({ ...run, status: 'success' as const }), + execute: jest.fn( + (run: DeployRunSummary): Promise => + Promise.resolve({ ...run, status: 'success' as const }), ), - syncJenkinsRun: jest.fn((run: DeployRunSummary) => - Promise.resolve({ ...run, status: 'success' as const }), + syncJenkinsRun: jest.fn( + (run: DeployRunSummary): Promise => + Promise.resolve({ ...run, status: 'success' as const }), ), - } satisfies Pick; + syncJenkinsRunFromCompletedLog: jest.fn( + (run: DeployRunSummary): Promise => + Promise.resolve(run), + ), + } satisfies Pick< + DeployExecutionService, + 'execute' | 'syncJenkinsRun' | 'syncJenkinsRunFromCompletedLog' + >; const jenkins = { healthSummary: jest.fn(() => ({ status: 'ok' as const })), cancelQueueItem: jest.fn(() => @@ -137,20 +146,14 @@ describe('DeployRunsService', () => { requested: true as const, }), ), - findBuildByQueueId: jest.fn( - (_jobPath: string, _queueId: string) => { - void _jobPath; - void _queueId; + findBuildByQueueId: jest.fn((_jobPath: string, _queueId: string) => { + void _jobPath; + void _queueId; - return Promise.resolve(undefined); - }, - ), + return Promise.resolve(undefined); + }), findLatestBuildByParameter: jest.fn( - ( - _jobPath: string, - _parameterName: string, - _parameterValue: string, - ) => { + (_jobPath: string, _parameterName: string, _parameterValue: string) => { void _jobPath; void _parameterName; void _parameterValue; @@ -188,6 +191,23 @@ describe('DeployRunsService', () => { AuthService, 'listAllowedProjectKeys' | 'requireProjectPermission' | 'assertSuperAdmin' >; + const service = new DeployRunsService( + repository as unknown as DeployRunRepository, + projects as unknown as ProjectsService, + audit as unknown as AuditService, + notification as unknown as DeployNotificationService, + execution as unknown as DeployExecutionService, + jenkins as unknown as JenkinsClient, + auth as unknown as AuthService, + ); + const waitBeforeRetryingJenkinsLogBuildLink = jest + .spyOn( + service as unknown as { + waitBeforeRetryingJenkinsLogBuildLink: () => Promise; + }, + 'waitBeforeRetryingJenkinsLogBuildLink', + ) + .mockResolvedValue(undefined); return { audit, @@ -197,15 +217,8 @@ describe('DeployRunsService', () => { notification, projects, repository, - service: new DeployRunsService( - repository as unknown as DeployRunRepository, - projects as unknown as ProjectsService, - audit as unknown as AuditService, - notification as unknown as DeployNotificationService, - execution as unknown as DeployExecutionService, - jenkins as unknown as JenkinsClient, - auth as unknown as AuthService, - ), + service, + waitBeforeRetryingJenkinsLogBuildLink, }; } @@ -432,6 +445,48 @@ describe('DeployRunsService', () => { expect(run).toBe(runningRun); }); + it('retries read-through Jenkins sync after a run version conflict', async () => { + const runningRun: DeployRunSummary = { + ...baseRun, + id: 'run_running', + status: 'running', + jenkinsBuildNumber: 7, + }; + const latestRun: DeployRunSummary = { + ...runningRun, + version: 2, + }; + const { execution, repository, service } = createService({ + runs: [runningRun], + }); + repository.findById + .mockResolvedValueOnce(runningRun) + .mockResolvedValueOnce(latestRun); + execution.syncJenkinsRun + .mockRejectedValueOnce( + new AppError( + 'CONFLICT', + '发布记录 run_running 已被其他流程更新，请刷新后重试', + 409, + ), + ) + .mockResolvedValueOnce({ ...latestRun, status: 'success' }); + + const run = await service.getRun('run_running'); + + expect(run.status).toBe('success'); + expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith( + 1, + runningRun, + expect.objectContaining({ key: runningRun.projectKey }), + ); + expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith( + 2, + latestRun, + expect.objectContaining({ key: runningRun.projectKey }), + ); + }); + it('links a missing Jenkins build before reading logs', async () => { const unlinkedRun: DeployRunSummary = { ...baseRun, @@ -484,6 +539,38 @@ describe('DeployRunsService', () => { ); }); + it('uses completed Jenkins logs to advance status while reading logs', async () => { + const runningRun: DeployRunSummary = { + ...baseRun, + id: 'run_log_finished', + status: 'running', + jenkinsBuildNumber: 8, + }; + const { execution, jenkins, service } = createService({ + runs: [runningRun], + }); + execution.syncJenkinsRun.mockResolvedValueOnce(runningRun); + jenkins.getProgressiveText.mockResolvedValueOnce({ + text: [ + 'deploy ok', + '[Pipeline] End of Pipeline', + 'Finished: SUCCESS', + ].join('\n'), + nextStart: 56, + hasMore: false, + }); + + const log = await service.getJenkinsLog('run_log_finished', undefined); + + expect(log.text).toContain('Finished: SUCCESS'); + expect(execution.syncJenkinsRunFromCompletedLog).toHaveBeenCalledWith( + runningRun, + expect.objectContaining({ key: 'access-manage' }), + expect.stringContaining('Finished: SUCCESS'), + false, + ); + }); + it('links a missing Jenkins build by queue id before reading logs', async () => { const unlinkedRun: DeployRunSummary = { ...baseRun, @@ -535,6 +622,47 @@ describe('DeployRunsService', () => { ); }); + it('waits until Jenkins has created a build before reading logs', async () => { + const unlinkedRun: DeployRunSummary = { + ...baseRun, + id: 'run_log_wait_queue_build', + status: 'running', + jenkinsQueueId: '439', + }; + const { jenkins, service, waitBeforeRetryingJenkinsLogBuildLink } = + createService({ + runs: [unlinkedRun], + }); + const findBuildByQueueId = + jenkins.findBuildByQueueId as unknown as jest.MockedFunction< + JenkinsClient['findBuildByQueueId'] + >; + findBuildByQueueId.mockResolvedValueOnce(undefined).mockResolvedValueOnce({ + jobPath: 'my-project-prod/prod-access-manage-tag', + buildNumber: 8, + building: true, + url: 'https://jenkins.example/job/8/', + queueId: '439', + }); + + const log = await service.getJenkinsLog( + 'run_log_wait_queue_build', + undefined, + ); + + expect(log.buildNumber).toBe(8); + expect(findBuildByQueueId).toHaveBeenCalledTimes(2); + expect(waitBeforeRetryingJenkinsLogBuildLink).toHaveBeenCalledTimes(1); + expect(jenkins.getProgressiveText).toHaveBeenCalledWith( + 'my-project-dev/test-access-manage-develop', + 8, + 0, + ); + expect( + waitBeforeRetryingJenkinsLogBuildLink.mock.invocationCallOrder[0], + ).toBeLessThan(jenkins.getProgressiveText.mock.invocationCallOrder[0]); + }); + it('cancels a queued Jenkins execution before marking the run canceled', async () => { const queuedRun: DeployRunSummary = { ...baseRun, @@ -625,9 +753,7 @@ describe('DeployRunsService', () => { it('rejects non-terminal retry requests before creating a new run', async () => { const { repository, service } = createService(); - await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf( - AppError, - ); + await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf(AppError); expect(repository.create).not.toHaveBeenCalled(); }); diff --git a/src/deploy-runs/deploy-runs.service.ts b/src/deploy-runs/deploy-runs.service.ts index df19a46..63359f5 100644 --- a/src/deploy-runs/deploy-runs.service.ts +++ b/src/deploy-runs/deploy-runs.service.ts @@ -31,6 +31,9 @@ type ReadRunsOptions = { syncJenkins?: boolean; }; +const jenkinsSyncConflictRetryCount = 2; +const jenkinsLogBuildLinkMaxAttempts = 5; +const jenkinsLogBuildLinkRetryDelayMs = 1000; const escapeControl = String.fromCharCode(27); const jenkinsConsoleNotePattern = new RegExp( `${escapeControl}\\[8mha:/{4}[A-Za-z0-9+/=]+${escapeControl}\\[0m`, @@ -274,7 +277,7 @@ export class DeployRunsService { const project = await this.projectsService.getProject(run.projectKey); - return this.deployExecutionService.syncJenkinsRun(run, project); + return this.syncJenkinsRunWithRetry(run, project); } async getJenkinsLog( @@ -288,7 +291,7 @@ export class DeployRunsService { run.jenkinsBuildNumber === undefined || run.jenkinsBuildNumber === null ) { - run = await this.linkMissingJenkinsBuildForLog(run); + run = await this.waitForJenkinsBuildForLog(run); } if ( @@ -306,17 +309,24 @@ export class DeployRunsService { ); } + const jenkinsBuildNumber = run.jenkinsBuildNumber; const project = await this.projectsService.getProject(run.projectKey); const environment = this.findEnvironment(project, run); const normalizedStart = this.normalizeLogStart(start); const log = await this.jenkinsClient.getProgressiveText( environment.jenkinsJobPath, - run.jenkinsBuildNumber, + jenkinsBuildNumber, normalizedStart, ); const text = redactSensitive({ text: this.cleanJenkinsLogText(log.text), }).text; + run = await this.syncJenkinsRunFromLogForRead( + run, + project, + text, + log.hasMore, + ); await this.auditService.record({ action: 'JENKINS_LOG_VIEWED', @@ -326,7 +336,7 @@ export class DeployRunsService { after: { projectKey: run.projectKey, environment: run.environment, - jenkinsBuildNumber: run.jenkinsBuildNumber, + jenkinsBuildNumber, start: normalizedStart, nextStart: log.nextStart, hasMore: log.hasMore, @@ -337,7 +347,7 @@ export class DeployRunsService { return { runId: run.id, jobPath: environment.jenkinsJobPath, - buildNumber: run.jenkinsBuildNumber, + buildNumber: jenkinsBuildNumber, start: normalizedStart, nextStart: log.nextStart, hasMore: log.hasMore, @@ -361,10 +371,7 @@ export class DeployRunsService { for (const run of candidates) { try { const project = await this.projectsService.getProject(run.projectKey); - const syncedRun = await this.deployExecutionService.syncJenkinsRun( - run, - project, - ); + const syncedRun = await this.syncJenkinsRunWithRetry(run, project); syncedRuns.push(syncedRun); } catch (error) { failedRuns.push({ @@ -443,7 +450,7 @@ export class DeployRunsService { try { const project = await this.projectsService.getProject(run.projectKey); - return await this.deployExecutionService.syncJenkinsRun(run, project); + return await this.syncJenkinsRunWithRetry(run, project); } catch { return run; } @@ -480,7 +487,9 @@ export class DeployRunsService { ? 'running' : run.status, jenkinsQueueId: - 'queueId' in matchedBuild ? matchedBuild.queueId : run.jenkinsQueueId, + 'queueId' in matchedBuild + ? matchedBuild.queueId + : run.jenkinsQueueId, jenkinsBuildNumber: matchedBuild.buildNumber, jenkinsBuildUrl: matchedBuild.url, }, @@ -508,15 +517,133 @@ export class DeployRunsService { }, }); - return await this.deployExecutionService.syncJenkinsRun( - linkedRun, + return await this.syncJenkinsRunWithRetry(linkedRun, project); + } catch { + return run; + } + } + + private async waitForJenkinsBuildForLog( + run: DeployRunSummary, + ): Promise { + let currentRun = run; + + for ( + let attempt = 0; + attempt < jenkinsLogBuildLinkMaxAttempts; + attempt += 1 + ) { + currentRun = await this.linkMissingJenkinsBuildForLog(currentRun); + + if ( + currentRun.jenkinsBuildNumber !== undefined && + currentRun.jenkinsBuildNumber !== null + ) { + return currentRun; + } + + if (attempt < jenkinsLogBuildLinkMaxAttempts - 1) { + await this.waitBeforeRetryingJenkinsLogBuildLink(); + } + } + + return currentRun; + } + + private waitBeforeRetryingJenkinsLogBuildLink(): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, jenkinsLogBuildLinkRetryDelayMs); + timer.unref?.(); + }); + } + + private async syncJenkinsRunFromLogForRead( + run: DeployRunSummary, + project: ProjectSummary, + logText: string, + hasMore: boolean, + ): Promise { + try { + return await this.syncJenkinsLogResultWithRetry( + run, project, + logText, + hasMore, ); } catch { return run; } } + private async syncJenkinsRunWithRetry( + run: DeployRunSummary, + project: ProjectSummary, + ): Promise { + return this.retryJenkinsSync(run, (currentRun) => + this.deployExecutionService.syncJenkinsRun(currentRun, project), + ); + } + + private async syncJenkinsLogResultWithRetry( + run: DeployRunSummary, + project: ProjectSummary, + logText: string, + hasMore: boolean, + ): Promise { + return this.retryJenkinsSync(run, (currentRun) => + this.deployExecutionService.syncJenkinsRunFromCompletedLog( + currentRun, + project, + logText, + hasMore, + ), + ); + } + + private async retryJenkinsSync( + run: DeployRunSummary, + operation: (currentRun: DeployRunSummary) => Promise, + ): Promise { + let currentRun = run; + + for ( + let attempt = 0; + attempt <= jenkinsSyncConflictRetryCount; + attempt += 1 + ) { + try { + return await operation(currentRun); + } catch (error) { + if ( + !this.isRunVersionConflict(error) || + attempt === jenkinsSyncConflictRetryCount + ) { + throw error; + } + + const latestRun = await this.deployRunRepository.findById( + currentRun.id, + ); + + if (!latestRun) { + throw error; + } + + if (!this.isJenkinsSyncCandidate(latestRun)) { + return latestRun; + } + + currentRun = latestRun; + } + } + + return currentRun; + } + + private isRunVersionConflict(error: unknown): boolean { + return error instanceof AppError && error.code === 'CONFLICT'; + } + private normalizeLogStart(start: string | undefined): number { if (start === undefined || start === '') { return 0;