devops-platform/devops-platform-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 同步 Jenkins 完成状态

Browse Source
This commit is contained in:
湛兮
2026-06-12 14:43:46 +08:00
parent 345fa67f3a
commit 1b645dd9c8
4 changed files with 480 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/deploy-runs/deploy-execution.service.spec.ts
+36
View File
@@ -301,6 +301,42 @@ describe('DeployExecutionService', () => {
);
});
it('recovers a completed Jenkins result from the console log when build status has no result yet', async () => {
const { audit, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
notificationStatus: 'sent',
queueId: 'queue_log_success',
queueBuildNumber: 22,
logText: [
'deploy ok',
'[Pipeline] End of Pipeline',
'Finished: SUCCESS',
].join('\n'),
});
const run = await service.execute(
await createRun(repository, 'run_log_success_001'),
project,
);
const result = await service.syncJenkinsRun(run, project);
expect(result.status).toBe('success');
expect(result.jenkinsBuildNumber).toBe(22);
expect(step(result, 'jenkins-build')?.status).toBe('success');
expect(step(result, 'health-check')?.status).toBe('success');
expect(step(result, 'notify-wecom')?.status).toBe('success');
expect(step(result, 'finish-release')?.status).toBe('success');
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({
action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG',
after: expect.objectContaining({
buildNumber: 22,
result: 'SUCCESS',
}),
}),
);
});
it('links a Jenkins build by queue id when the queue item has expired', async () => {
const { audit, jenkins, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
src/deploy-runs/deploy-execution.service.ts
+145 -11
View File
@@ -24,6 +24,8 @@ import {
import { DeployRunRepository } from './deploy-run.repository';
import { DeployRunStepStatus, DeployRunSummary } from './deploy-run.types';
type JenkinsTerminalResult = NonNullable<JenkinsBuildStatus['result']>;
@Injectable()
export class DeployExecutionService {
constructor(
@@ -206,6 +208,21 @@ export class DeployExecutionService {
);
if (buildStatus.building || !buildStatus.result) {
if (!buildStatus.building) {
const recoveredResult = await this.recoverCompletedBuildResultFromLog(
currentRun,
environment.jenkinsJobPath,
);
if (recoveredResult) {
return this.completeJenkinsRunWithResult(
currentRun,
recoveredResult,
environment.jenkinsJobPath,
);
}
}
return this.markStep(
currentRun,
'jenkins-build',
@@ -214,17 +231,44 @@ export class DeployExecutionService {
);
}
if (buildStatus.result === 'SUCCESS') {
return this.completeSuccessfulJenkinsRun(currentRun);
}
return this.failJenkinsRun(
return this.completeJenkinsRunWithResult(
currentRun,
buildStatus.result,
environment.jenkinsJobPath,
);
}
async syncJenkinsRunFromCompletedLog(
run: DeployRunSummary,
project: ProjectSummary,
logText: string,
hasMore: boolean,
): Promise<DeployRunSummary> {
if (['success', 'failed', 'canceled'].includes(run.status) || hasMore) {
return run;
}
const result = this.extractJenkinsFinishedResult(logText);
if (!result) {
return run;
}
const environment = this.findEnvironment(project, run);
await this.recordJenkinsLogResultRecovered(
run,
result,
environment.jenkinsJobPath,
);
return this.completeJenkinsRunWithResult(
run,
result,
environment.jenkinsJobPath,
logText,
);
}
private async resolveJenkinsBuildReference(
run: DeployRunSummary,
environment: ProjectEnvironmentSummary,
@@ -673,10 +717,13 @@ export class DeployExecutionService {
private async failJenkinsRun(
run: DeployRunSummary,
result: Exclude<NonNullable<JenkinsBuildStatus['result']>, 'SUCCESS'>,
result: Exclude<JenkinsTerminalResult, 'SUCCESS'>,
jobPath: string,
logText?: string,
): Promise<DeployRunSummary> {
const logExcerpt = await this.readFailureLogExcerpt(run, jobPath);
const logExcerpt = logText
? this.toLogExcerpt(logText)
: await this.readFailureLogExcerpt(run, jobPath);
const failureSummary = `Jenkins 构建 #${run.jenkinsBuildNumber ?? '-'} ${result}`;
let currentRun =
(await this.deployRunRepository.updateStep(
@@ -729,7 +776,97 @@ export class DeployExecutionService {
jobPath,
run.jenkinsBuildNumber,
);
const logExcerpt = log.text
return this.toLogExcerpt(log.text);
} catch {
return undefined;
}
}
private async recoverCompletedBuildResultFromLog(
run: DeployRunSummary,
jobPath: string,
): Promise<JenkinsTerminalResult | undefined> {
if (!run.jenkinsBuildNumber) {
return undefined;
}
try {
const log = await this.jenkinsClient.getProgressiveText(
jobPath,
run.jenkinsBuildNumber,
);
if (log.hasMore) {
return undefined;
}
const result = this.extractJenkinsFinishedResult(log.text);
if (!result) {
return undefined;
}
await this.recordJenkinsLogResultRecovered(run, result, jobPath);
return result;
} catch {
return undefined;
}
}
private async recordJenkinsLogResultRecovered(
run: DeployRunSummary,
result: JenkinsTerminalResult,
jobPath: string,
): Promise<void> {
await this.auditService.record({
action: 'JENKINS_BUILD_RESULT_RECOVERED_FROM_LOG',
resourceType: 'deploy_run',
resourceId: run.id,
actorName: run.operator,
after: {
jobPath,
buildNumber: run.jenkinsBuildNumber,
result,
},
});
}
private completeJenkinsRunWithResult(
run: DeployRunSummary,
result: JenkinsTerminalResult,
jobPath: string,
logText?: string,
): Promise<DeployRunSummary> {
if (result === 'SUCCESS') {
return this.completeSuccessfulJenkinsRun(run);
}
return this.failJenkinsRun(run, result, jobPath, logText);
}
private extractJenkinsFinishedResult(
logText: string,
): JenkinsTerminalResult | undefined {
const match = logText.match(
/(?:^|\r?\n)\s*Finished:\s*(SUCCESS|FAILURE|ABORTED|UNSTABLE)\b/i,
);
const result = match?.[1]?.toUpperCase();
if (
result === 'SUCCESS' ||
result === 'FAILURE' ||
result === 'ABORTED' ||
result === 'UNSTABLE'
) {
return result;
}
return undefined;
}
private toLogExcerpt(logText: string): string {
const logExcerpt = logText
.split(/\r?\n/)
.filter(Boolean)
.slice(-24)
@@ -737,9 +874,6 @@ export class DeployExecutionService {
.slice(-4000);
return redactSensitive({ logExcerpt }).logExcerpt;
} catch {
return undefined;
}
}
private findEnvironment(
src/deploy-runs/deploy-runs.service.spec.ts
+150 -24
View File
@@ -114,13 +114,22 @@ describe('DeployRunsService', () => {
),
} satisfies Pick<DeployNotificationService, 'sendDeployRunEvent'>;
const execution = {
execute: jest.fn((run: DeployRunSummary) =>
execute: jest.fn(
(run: DeployRunSummary): Promise<DeployRunSummary> =>
Promise.resolve({ ...run, status: 'success' as const }),
),
syncJenkinsRun: jest.fn((run: DeployRunSummary) =>
syncJenkinsRun: jest.fn(
(run: DeployRunSummary): Promise<DeployRunSummary> =>
Promise.resolve({ ...run, status: 'success' as const }),
),
} satisfies Pick<DeployExecutionService, 'execute' | 'syncJenkinsRun'>;
syncJenkinsRunFromCompletedLog: jest.fn(
(run: DeployRunSummary): Promise<DeployRunSummary> =>
Promise.resolve(run),
),
} satisfies Pick<
DeployExecutionService,
'execute' | 'syncJenkinsRun' | 'syncJenkinsRunFromCompletedLog'
>;
const jenkins = {
healthSummary: jest.fn(() => ({ status: 'ok' as const })),
cancelQueueItem: jest.fn(() =>
@@ -137,20 +146,14 @@ describe('DeployRunsService', () => {
requested: true as const,
}),
),
findBuildByQueueId: jest.fn(
(_jobPath: string, _queueId: string) => {
findBuildByQueueId: jest.fn((_jobPath: string, _queueId: string) => {
void _jobPath;
void _queueId;
return Promise.resolve(undefined);
},
),
}),
findLatestBuildByParameter: jest.fn(
(
_jobPath: string,
_parameterName: string,
_parameterValue: string,
) => {
(_jobPath: string, _parameterName: string, _parameterValue: string) => {
void _jobPath;
void _parameterName;
void _parameterValue;
@@ -188,6 +191,23 @@ describe('DeployRunsService', () => {
AuthService,
'listAllowedProjectKeys' | 'requireProjectPermission' | 'assertSuperAdmin'
>;
const service = new DeployRunsService(
repository as unknown as DeployRunRepository,
projects as unknown as ProjectsService,
audit as unknown as AuditService,
notification as unknown as DeployNotificationService,
execution as unknown as DeployExecutionService,
jenkins as unknown as JenkinsClient,
auth as unknown as AuthService,
);
const waitBeforeRetryingJenkinsLogBuildLink = jest
.spyOn(
service as unknown as {
waitBeforeRetryingJenkinsLogBuildLink: () => Promise<void>;
},
'waitBeforeRetryingJenkinsLogBuildLink',
)
.mockResolvedValue(undefined);
return {
audit,
@@ -197,15 +217,8 @@ describe('DeployRunsService', () => {
notification,
projects,
repository,
service: new DeployRunsService(
repository as unknown as DeployRunRepository,
projects as unknown as ProjectsService,
audit as unknown as AuditService,
notification as unknown as DeployNotificationService,
execution as unknown as DeployExecutionService,
jenkins as unknown as JenkinsClient,
auth as unknown as AuthService,
),
service,
waitBeforeRetryingJenkinsLogBuildLink,
};
}
@@ -432,6 +445,48 @@ describe('DeployRunsService', () => {
expect(run).toBe(runningRun);
});
it('retries read-through Jenkins sync after a run version conflict', async () => {
const runningRun: DeployRunSummary = {
...baseRun,
id: 'run_running',
status: 'running',
jenkinsBuildNumber: 7,
};
const latestRun: DeployRunSummary = {
...runningRun,
version: 2,
};
const { execution, repository, service } = createService({
runs: [runningRun],
});
repository.findById
.mockResolvedValueOnce(runningRun)
.mockResolvedValueOnce(latestRun);
execution.syncJenkinsRun
.mockRejectedValueOnce(
new AppError(
'CONFLICT',
'发布记录 run_running 已被其他流程更新，请刷新后重试',
409,
),
)
.mockResolvedValueOnce({ ...latestRun, status: 'success' });
const run = await service.getRun('run_running');
expect(run.status).toBe('success');
expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith(
1,
runningRun,
expect.objectContaining({ key: runningRun.projectKey }),
);
expect(execution.syncJenkinsRun).toHaveBeenNthCalledWith(
2,
latestRun,
expect.objectContaining({ key: runningRun.projectKey }),
);
});
it('links a missing Jenkins build before reading logs', async () => {
const unlinkedRun: DeployRunSummary = {
...baseRun,
@@ -484,6 +539,38 @@ describe('DeployRunsService', () => {
);
});
it('uses completed Jenkins logs to advance status while reading logs', async () => {
const runningRun: DeployRunSummary = {
...baseRun,
id: 'run_log_finished',
status: 'running',
jenkinsBuildNumber: 8,
};
const { execution, jenkins, service } = createService({
runs: [runningRun],
});
execution.syncJenkinsRun.mockResolvedValueOnce(runningRun);
jenkins.getProgressiveText.mockResolvedValueOnce({
text: [
'deploy ok',
'[Pipeline] End of Pipeline',
'Finished: SUCCESS',
].join('\n'),
nextStart: 56,
hasMore: false,
});
const log = await service.getJenkinsLog('run_log_finished', undefined);
expect(log.text).toContain('Finished: SUCCESS');
expect(execution.syncJenkinsRunFromCompletedLog).toHaveBeenCalledWith(
runningRun,
expect.objectContaining({ key: 'access-manage' }),
expect.stringContaining('Finished: SUCCESS'),
false,
);
});
it('links a missing Jenkins build by queue id before reading logs', async () => {
const unlinkedRun: DeployRunSummary = {
...baseRun,
@@ -535,6 +622,47 @@ describe('DeployRunsService', () => {
);
});
it('waits until Jenkins has created a build before reading logs', async () => {
const unlinkedRun: DeployRunSummary = {
...baseRun,
id: 'run_log_wait_queue_build',
status: 'running',
jenkinsQueueId: '439',
};
const { jenkins, service, waitBeforeRetryingJenkinsLogBuildLink } =
createService({
runs: [unlinkedRun],
});
const findBuildByQueueId =
jenkins.findBuildByQueueId as unknown as jest.MockedFunction<
JenkinsClient['findBuildByQueueId']
>;
findBuildByQueueId.mockResolvedValueOnce(undefined).mockResolvedValueOnce({
jobPath: 'my-project-prod/prod-access-manage-tag',
buildNumber: 8,
building: true,
url: 'https://jenkins.example/job/8/',
queueId: '439',
});
const log = await service.getJenkinsLog(
'run_log_wait_queue_build',
undefined,
);
expect(log.buildNumber).toBe(8);
expect(findBuildByQueueId).toHaveBeenCalledTimes(2);
expect(waitBeforeRetryingJenkinsLogBuildLink).toHaveBeenCalledTimes(1);
expect(jenkins.getProgressiveText).toHaveBeenCalledWith(
'my-project-dev/test-access-manage-develop',
8,
0,
);
expect(
waitBeforeRetryingJenkinsLogBuildLink.mock.invocationCallOrder[0],
).toBeLessThan(jenkins.getProgressiveText.mock.invocationCallOrder[0]);
});
it('cancels a queued Jenkins execution before marking the run canceled', async () => {
const queuedRun: DeployRunSummary = {
...baseRun,
@@ -625,9 +753,7 @@ describe('DeployRunsService', () => {
it('rejects non-terminal retry requests before creating a new run', async () => {
const { repository, service } = createService();
await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf(
AppError,
);
await expect(service.retryRun(baseRun.id)).rejects.toBeInstanceOf(AppError);
expect(repository.create).not.toHaveBeenCalled();
});
src/deploy-runs/deploy-runs.service.ts
+140 -13
View File
@@ -31,6 +31,9 @@ type ReadRunsOptions = {
syncJenkins?: boolean;
};
const jenkinsSyncConflictRetryCount = 2;
const jenkinsLogBuildLinkMaxAttempts = 5;
const jenkinsLogBuildLinkRetryDelayMs = 1000;
const escapeControl = String.fromCharCode(27);
const jenkinsConsoleNotePattern = new RegExp(
`${escapeControl}\\[8mha:/{4}[A-Za-z0-9+/=]+${escapeControl}\\[0m`,
@@ -274,7 +277,7 @@ export class DeployRunsService {
const project = await this.projectsService.getProject(run.projectKey);
return this.deployExecutionService.syncJenkinsRun(run, project);
return this.syncJenkinsRunWithRetry(run, project);
}
async getJenkinsLog(
@@ -288,7 +291,7 @@ export class DeployRunsService {
run.jenkinsBuildNumber === undefined ||
run.jenkinsBuildNumber === null
) {
run = await this.linkMissingJenkinsBuildForLog(run);
run = await this.waitForJenkinsBuildForLog(run);
}
if (
@@ -306,17 +309,24 @@ export class DeployRunsService {
);
}
const jenkinsBuildNumber = run.jenkinsBuildNumber;
const project = await this.projectsService.getProject(run.projectKey);
const environment = this.findEnvironment(project, run);
const normalizedStart = this.normalizeLogStart(start);
const log = await this.jenkinsClient.getProgressiveText(
environment.jenkinsJobPath,
run.jenkinsBuildNumber,
jenkinsBuildNumber,
normalizedStart,
);
const text = redactSensitive({
text: this.cleanJenkinsLogText(log.text),
}).text;
run = await this.syncJenkinsRunFromLogForRead(
run,
project,
text,
log.hasMore,
);
await this.auditService.record({
action: 'JENKINS_LOG_VIEWED',
@@ -326,7 +336,7 @@ export class DeployRunsService {
after: {
projectKey: run.projectKey,
environment: run.environment,
jenkinsBuildNumber: run.jenkinsBuildNumber,
jenkinsBuildNumber,
start: normalizedStart,
nextStart: log.nextStart,
hasMore: log.hasMore,
@@ -337,7 +347,7 @@ export class DeployRunsService {
return {
runId: run.id,
jobPath: environment.jenkinsJobPath,
buildNumber: run.jenkinsBuildNumber,
buildNumber: jenkinsBuildNumber,
start: normalizedStart,
nextStart: log.nextStart,
hasMore: log.hasMore,
@@ -361,10 +371,7 @@ export class DeployRunsService {
for (const run of candidates) {
try {
const project = await this.projectsService.getProject(run.projectKey);
const syncedRun = await this.deployExecutionService.syncJenkinsRun(
run,
project,
);
const syncedRun = await this.syncJenkinsRunWithRetry(run, project);
syncedRuns.push(syncedRun);
} catch (error) {
failedRuns.push({
@@ -443,7 +450,7 @@ export class DeployRunsService {
try {
const project = await this.projectsService.getProject(run.projectKey);
return await this.deployExecutionService.syncJenkinsRun(run, project);
return await this.syncJenkinsRunWithRetry(run, project);
} catch {
return run;
}
@@ -480,7 +487,9 @@ export class DeployRunsService {
? 'running'
: run.status,
jenkinsQueueId:
'queueId' in matchedBuild ? matchedBuild.queueId : run.jenkinsQueueId,
'queueId' in matchedBuild
? matchedBuild.queueId
: run.jenkinsQueueId,
jenkinsBuildNumber: matchedBuild.buildNumber,
jenkinsBuildUrl: matchedBuild.url,
},
@@ -508,15 +517,133 @@ export class DeployRunsService {
},
});
return await this.deployExecutionService.syncJenkinsRun(
linkedRun,
return await this.syncJenkinsRunWithRetry(linkedRun, project);
} catch {
return run;
}
}
private async waitForJenkinsBuildForLog(
run: DeployRunSummary,
): Promise<DeployRunSummary> {
let currentRun = run;
for (
let attempt = 0;
attempt < jenkinsLogBuildLinkMaxAttempts;
attempt += 1
) {
currentRun = await this.linkMissingJenkinsBuildForLog(currentRun);
if (
currentRun.jenkinsBuildNumber !== undefined &&
currentRun.jenkinsBuildNumber !== null
) {
return currentRun;
}
if (attempt < jenkinsLogBuildLinkMaxAttempts - 1) {
await this.waitBeforeRetryingJenkinsLogBuildLink();
}
}
return currentRun;
}
private waitBeforeRetryingJenkinsLogBuildLink(): Promise<void> {
return new Promise((resolve) => {
const timer = setTimeout(resolve, jenkinsLogBuildLinkRetryDelayMs);
timer.unref?.();
});
}
private async syncJenkinsRunFromLogForRead(
run: DeployRunSummary,
project: ProjectSummary,
logText: string,
hasMore: boolean,
): Promise<DeployRunSummary> {
try {
return await this.syncJenkinsLogResultWithRetry(
run,
project,
logText,
hasMore,
);
} catch {
return run;
}
}
private async syncJenkinsRunWithRetry(
run: DeployRunSummary,
project: ProjectSummary,
): Promise<DeployRunSummary> {
return this.retryJenkinsSync(run, (currentRun) =>
this.deployExecutionService.syncJenkinsRun(currentRun, project),
);
}
private async syncJenkinsLogResultWithRetry(
run: DeployRunSummary,
project: ProjectSummary,
logText: string,
hasMore: boolean,
): Promise<DeployRunSummary> {
return this.retryJenkinsSync(run, (currentRun) =>
this.deployExecutionService.syncJenkinsRunFromCompletedLog(
currentRun,
project,
logText,
hasMore,
),
);
}
private async retryJenkinsSync(
run: DeployRunSummary,
operation: (currentRun: DeployRunSummary) => Promise<DeployRunSummary>,
): Promise<DeployRunSummary> {
let currentRun = run;
for (
let attempt = 0;
attempt <= jenkinsSyncConflictRetryCount;
attempt += 1
) {
try {
return await operation(currentRun);
} catch (error) {
if (
!this.isRunVersionConflict(error) ||
attempt === jenkinsSyncConflictRetryCount
) {
throw error;
}
const latestRun = await this.deployRunRepository.findById(
currentRun.id,
);
if (!latestRun) {
throw error;
}
if (!this.isJenkinsSyncCandidate(latestRun)) {
return latestRun;
}
currentRun = latestRun;
}
}
return currentRun;
}
private isRunVersionConflict(error: unknown): boolean {
return error instanceof AppError && error.code === 'CONFLICT';
}
private normalizeLogStart(start: string | undefined): number {
if (start === undefined || start === '') {
return 0;