fix: 完善 Jenkins 构建追踪

This commit is contained in:
湛兮
2026-06-12 12:26:28 +08:00
parent c2afe6e693
commit 345fa67f3a
8 changed files with 1255 additions and 168 deletions
+1
View File
@@ -114,6 +114,7 @@ pnpm reset:super-admin-password
- `PRISMA_CONNECT_ON_BOOT`:启动时是否主动连接数据库。
- `REDIS_URL`Redis 连接地址,`/health` 会执行真实 PINGJenkins 自动同步和通知 outbox 重试会优先使用 BullMQ 调度。
- `JENKINS_BASE_URL``JENKINS_USERNAME``JENKINS_API_TOKEN`Jenkins 集成配置。
- Jenkins 配置缺失时,发布单会在 Jenkins 步骤失败,不会模拟成功;生产环境需要提供完整 Jenkins 配置,并建议开启 `JENKINS_AUTO_SYNC_ENABLED=true` 持续同步队列和构建状态。
- `GITEA_BASE_URL``GITEA_TOKEN``GITEA_WEBHOOK_SECRET``GITEA_WEBHOOK_RELAY_BASE_URL`Gitea 集成与 Jenkins relay 诊断配置。
- `NOTIFICATION_PROVIDER``WECOM_WEBHOOK_URL``FEISHU_WEBHOOK_URL``NOTIFICATION_WEBHOOK_URL`:通知集成配置。
- `NOTIFICATION_OUTBOX_RETRY_ENABLED``NOTIFICATION_OUTBOX_RETRY_INTERVAL_MS``NOTIFICATION_OUTBOX_RETRY_DELAY_MS``NOTIFICATION_OUTBOX_MAX_ATTEMPTS``NOTIFICATION_OUTBOX_BATCH_SIZE`:通知 outbox 自动重试配置;配置 `REDIS_URL` 时使用 BullMQ 调度,否则降级为进程内定时器。
@@ -1,4 +1,5 @@
import { AuditService } from '../audit/audit.service';
import { IntegrationRequestError } from '../common/errors/integration-error';
import { JenkinsClient } from '../integrations/jenkins/jenkins.client';
import { DeployNotificationService } from '../notifications/wecom/deploy-notification.service';
import { ProjectSummary } from '../projects/project.types';
@@ -45,6 +46,10 @@ describe('DeployExecutionService', () => {
notificationStatus: 'sent' | 'skipped';
queueId?: string;
queueBuildNumber?: number;
queueSearchBuildNumber?: number;
jobBuildable?: boolean;
jobInfoError?: Error;
matchedBuildNumber?: number;
buildResult?: 'SUCCESS' | 'FAILURE' | 'ABORTED' | 'UNSTABLE';
buildRunning?: boolean;
logText?: string;
@@ -72,6 +77,19 @@ describe('DeployExecutionService', () => {
queueId: options.queueId ?? 'queue_001',
}),
),
getJobInfo: jest.fn(() => {
if (options.jobInfoError) {
return Promise.reject(options.jobInfoError);
}
return Promise.resolve({
jobPath: 'my-project-dev/test-access-manage-develop',
name: 'test-access-manage-develop',
fullName: 'my-project-dev/test-access-manage-develop',
url: 'https://jenkins.example/job/my-project-dev/job/test-access-manage-develop/',
buildable: options.jobBuildable ?? true,
});
}),
getQueueItem: jest.fn((queueId: string) =>
Promise.resolve({
queueId,
@@ -81,6 +99,35 @@ describe('DeployExecutionService', () => {
: undefined,
}),
),
findBuildByQueueId: jest.fn((_jobPath: string, queueId: string) =>
Promise.resolve(
options.queueSearchBuildNumber
? {
jobPath: 'my-project-dev/test-access-manage-develop',
buildNumber: options.queueSearchBuildNumber,
building: options.buildRunning ?? false,
result: options.buildResult,
url: `https://jenkins.example/job/${options.queueSearchBuildNumber}/`,
queueId,
}
: undefined,
),
),
findLatestBuildByParameter: jest.fn((_jobPath: string, name: string) =>
Promise.resolve(
options.matchedBuildNumber
? {
jobPath: 'my-project-dev/test-access-manage-develop',
buildNumber: options.matchedBuildNumber,
building: options.buildRunning ?? false,
result: options.buildResult,
url: `https://jenkins.example/job/${options.matchedBuildNumber}/`,
matchedParameterName: name,
matchedParameterValue: 'run_001',
}
: undefined,
),
),
getBuildStatus: jest.fn((_jobPath: string, buildNumber: number) =>
Promise.resolve({
jobPath: 'my-project-dev/test-access-manage-develop',
@@ -101,7 +148,10 @@ describe('DeployExecutionService', () => {
JenkinsClient,
| 'healthSummary'
| 'buildWithParameters'
| 'getJobInfo'
| 'getQueueItem'
| 'findBuildByQueueId'
| 'findLatestBuildByParameter'
| 'getBuildStatus'
| 'getProgressiveText'
>;
@@ -137,7 +187,7 @@ describe('DeployExecutionService', () => {
return run.steps.find((item) => item.bpmnNodeId === bpmnNodeId);
}
it('completes a dry-run release when Jenkins is not configured', async () => {
it('fails a release when Jenkins is not configured instead of dry-running success', async () => {
const { audit, jenkins, repository, service } = createService({
jenkinsHealth: {
status: 'not_configured',
@@ -149,20 +199,41 @@ describe('DeployExecutionService', () => {
const result = await service.execute(run, project);
expect(result.status).toBe('success');
expect(result.status).toBe('failed');
expect(result.jenkinsQueueId).toBeUndefined();
expect(result.failureSummary).toContain('Jenkins 集成尚未配置');
expect(step(result, 'gitea-check')?.status).toBe('success');
expect(step(result, 'jenkins-build')?.status).toBe('success');
expect(step(result, 'health-check')?.status).toBe('success');
expect(step(result, 'jenkins-build')?.status).toBe('failed');
expect(step(result, 'jenkins-build')?.errorSummary).toContain(
'Jenkins 集成尚未配置',
);
expect(step(result, 'health-check')?.status).toBe('pending');
expect(step(result, 'notify-wecom')?.status).toBe('skipped');
expect(step(result, 'finish-release')?.status).toBe('success');
expect(step(result, 'finish-release')?.status).toBe('pending');
expect(jenkins.buildWithParameters).not.toHaveBeenCalled();
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'JENKINS_DRY_RUN_COMPLETED' }),
expect.objectContaining({ action: 'DEPLOY_FAILED' }),
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'DEPLOY_SUCCEEDED' }),
});
it('fails before triggering when the configured Jenkins job is missing', async () => {
const { jenkins, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
notificationStatus: 'skipped',
jobInfoError: new IntegrationRequestError('jenkins', 'Jenkins 请求失败', {
status: 404,
}),
});
const run = await createRun(repository, 'run_missing_job_001');
const result = await service.execute(run, project);
expect(result.status).toBe('failed');
expect(result.failureSummary).toBe(
'Jenkins job 不存在:my-project-dev/test-access-manage-develop',
);
expect(step(result, 'jenkins-build')?.status).toBe('failed');
expect(jenkins.buildWithParameters).not.toHaveBeenCalled();
});
it('triggers Jenkins and keeps the run queued until status polling completes', async () => {
@@ -230,6 +301,58 @@ describe('DeployExecutionService', () => {
);
});
it('links a Jenkins build by queue id when the queue item has expired', async () => {
const { audit, jenkins, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
notificationStatus: 'sent',
queueId: 'queue_431',
queueSearchBuildNumber: 19,
buildResult: 'SUCCESS',
});
const run = await service.execute(
await createRun(repository, 'run_queue_search_001'),
project,
);
const result = await service.syncJenkinsRun(run, project);
expect(result.status).toBe('success');
expect(result.jenkinsBuildNumber).toBe(19);
expect(jenkins.findBuildByQueueId).toHaveBeenCalledWith(
'my-project-dev/test-access-manage-develop',
'queue_431',
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'JENKINS_BUILD_LINKED' }),
);
});
it('links an existing Jenkins build by DEPLOY_RUN_ID when the run has no build number', async () => {
const { audit, jenkins, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
notificationStatus: 'sent',
matchedBuildNumber: 21,
buildResult: 'SUCCESS',
});
const run = await createRun(repository, 'run_missing_build_001');
const result = await service.syncJenkinsRun(run, project);
expect(result.status).toBe('success');
expect(result.jenkinsBuildNumber).toBe(21);
expect(jenkins.findLatestBuildByParameter).toHaveBeenCalledWith(
'my-project-dev/test-access-manage-develop',
'DEPLOY_RUN_ID',
'run_missing_build_001',
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'JENKINS_BUILD_LINKED' }),
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'DEPLOY_SUCCEEDED' }),
);
});
it('syncs a queued Jenkins run to failure with a log excerpt', async () => {
const { audit, repository, service } = createService({
jenkinsHealth: { status: 'ok' },
+333 -112
View File
@@ -5,9 +5,14 @@
import { Injectable } from '@nestjs/common';
import { AuditService } from '../audit/audit.service';
import { AppError } from '../common/errors/app-error';
import { IntegrationRequestError } from '../common/errors/integration-error';
import { redactSensitive } from '../common/security/redact-sensitive';
import { JenkinsClient } from '../integrations/jenkins/jenkins.client';
import { JenkinsBuildStatus } from '../integrations/jenkins/jenkins.types';
import {
JenkinsBuildParameterMatch,
JenkinsBuildQueueMatch,
JenkinsBuildStatus,
} from '../integrations/jenkins/jenkins.types';
import {
DeployNotificationEvent,
DeployNotificationService,
@@ -61,11 +66,7 @@ export class DeployExecutionService {
try {
currentStep = 'gitea-check';
currentRun = await this.completeStep(
currentRun,
currentStep,
'success',
);
currentRun = await this.completeStep(currentRun, currentStep, 'success');
currentStep = 'jenkins-build';
const jenkinsResult = await this.advanceJenkinsBuild(currentRun, project);
@@ -77,21 +78,13 @@ export class DeployExecutionService {
}
currentStep = 'health-check';
currentRun = await this.completeStep(
currentRun,
currentStep,
'success',
);
currentRun = await this.completeStep(currentRun, currentStep, 'success');
currentStep = 'notify-wecom';
currentRun = await this.advanceNotification(currentRun, 'succeeded');
currentStep = 'finish-release';
currentRun = await this.completeStep(
currentRun,
currentStep,
'success',
);
currentRun = await this.completeStep(currentRun, currentStep, 'success');
currentRun = this.requireRun(
currentRun.id,
await this.deployRunRepository.updateRun(
@@ -169,66 +162,22 @@ export class DeployExecutionService {
return run;
}
if (!run.jenkinsQueueId && !run.jenkinsBuildNumber) {
return run;
}
const environment = this.findEnvironment(project, run);
let currentRun = run;
let buildNumber = currentRun.jenkinsBuildNumber;
if (!buildNumber && currentRun.jenkinsQueueId) {
const queueItem = await this.jenkinsClient.getQueueItem(
currentRun.jenkinsQueueId,
if (!buildNumber) {
const resolved = await this.resolveJenkinsBuildReference(
currentRun,
environment,
);
if (!queueItem.buildNumber) {
return this.requireRun(
currentRun.id,
await this.deployRunRepository.updateRun(
currentRun.id,
{
status: 'queued',
jenkinsQueueId: queueItem.queueId,
jenkinsBuildUrl: queueItem.executableUrl,
},
currentRun.version,
),
);
if (!resolved) {
return currentRun;
}
buildNumber = queueItem.buildNumber;
currentRun = this.requireRun(
currentRun.id,
await this.deployRunRepository.updateRun(
currentRun.id,
{
status: 'running',
jenkinsQueueId: queueItem.queueId,
jenkinsBuildNumber: queueItem.buildNumber,
jenkinsBuildUrl: queueItem.executableUrl,
},
currentRun.version,
),
);
currentRun = await this.markStep(
currentRun,
'jenkins-build',
'running',
`Jenkins 构建 #${queueItem.buildNumber} 正在运行。`,
);
await this.auditService.record({
action: 'JENKINS_BUILD_STARTED',
resourceType: 'deploy_run',
resourceId: currentRun.id,
actorName: currentRun.operator,
after: {
queueId: queueItem.queueId,
buildNumber: queueItem.buildNumber,
buildUrl: queueItem.executableUrl,
},
});
currentRun = resolved.run;
buildNumber = resolved.buildNumber;
}
if (!buildNumber) {
@@ -276,37 +225,254 @@ export class DeployExecutionService {
);
}
private async resolveJenkinsBuildReference(
run: DeployRunSummary,
environment: ProjectEnvironmentSummary,
): Promise<{ run: DeployRunSummary; buildNumber: number } | undefined> {
let queueLookupError: unknown;
if (run.jenkinsQueueId) {
try {
const queueItem = await this.jenkinsClient.getQueueItem(
run.jenkinsQueueId,
);
if (!queueItem.buildNumber) {
const matchedQueueBuild = await this.findBuildByQueueId(
environment,
queueItem.queueId,
);
if (matchedQueueBuild) {
const linkedRun = await this.recordMatchedJenkinsQueueBuild(
run,
matchedQueueBuild,
);
return {
run: linkedRun,
buildNumber: matchedQueueBuild.buildNumber,
};
}
this.requireRun(
run.id,
await this.deployRunRepository.updateRun(
run.id,
{
status: 'queued',
jenkinsQueueId: queueItem.queueId,
jenkinsBuildUrl: queueItem.executableUrl,
},
run.version,
),
);
return undefined;
}
const linkedRun = await this.recordJenkinsBuildStarted(run, {
buildNumber: queueItem.buildNumber,
buildUrl: queueItem.executableUrl,
queueId: queueItem.queueId,
source: 'queue',
});
return {
run: linkedRun,
buildNumber: queueItem.buildNumber,
};
} catch (error) {
queueLookupError = error;
const matchedQueueBuild = await this.findBuildByQueueId(
environment,
run.jenkinsQueueId,
);
if (matchedQueueBuild) {
const linkedRun = await this.recordMatchedJenkinsQueueBuild(
run,
matchedQueueBuild,
);
return {
run: linkedRun,
buildNumber: matchedQueueBuild.buildNumber,
};
}
}
}
const matchedBuild = await this.jenkinsClient.findLatestBuildByParameter(
environment.jenkinsJobPath,
'DEPLOY_RUN_ID',
run.id,
);
if (!matchedBuild) {
if (queueLookupError) {
throw queueLookupError instanceof Error
? queueLookupError
: new Error('Jenkins queue lookup failed');
}
return undefined;
}
const linkedRun = await this.recordMatchedJenkinsBuild(run, matchedBuild);
return {
run: linkedRun,
buildNumber: matchedBuild.buildNumber,
};
}
private async recordJenkinsBuildStarted(
run: DeployRunSummary,
input: {
buildNumber: number;
buildUrl?: string;
queueId?: string;
source: 'queue' | 'queue_search' | 'parameter_search';
},
): Promise<DeployRunSummary> {
let currentRun = this.requireRun(
run.id,
await this.deployRunRepository.updateRun(
run.id,
{
status: 'running',
jenkinsQueueId: input.queueId ?? run.jenkinsQueueId,
jenkinsBuildNumber: input.buildNumber,
jenkinsBuildUrl: input.buildUrl,
},
run.version,
),
);
currentRun = await this.markStep(
currentRun,
'jenkins-build',
'running',
`Jenkins 构建 #${input.buildNumber} 正在运行。`,
);
await this.auditService.record({
action:
input.source === 'queue'
? 'JENKINS_BUILD_STARTED'
: 'JENKINS_BUILD_LINKED',
resourceType: 'deploy_run',
resourceId: currentRun.id,
actorName: currentRun.operator,
after: {
queueId: input.queueId,
buildNumber: input.buildNumber,
buildUrl: input.buildUrl,
source: input.source,
},
});
return currentRun;
}
private async recordMatchedJenkinsQueueBuild(
run: DeployRunSummary,
matchedBuild: JenkinsBuildQueueMatch,
): Promise<DeployRunSummary> {
if (matchedBuild.building || !matchedBuild.result) {
return this.recordJenkinsBuildStarted(run, {
buildNumber: matchedBuild.buildNumber,
buildUrl: matchedBuild.url,
queueId: matchedBuild.queueId,
source: 'queue_search',
});
}
const currentRun = this.requireRun(
run.id,
await this.deployRunRepository.updateRun(
run.id,
{
jenkinsQueueId: matchedBuild.queueId,
jenkinsBuildNumber: matchedBuild.buildNumber,
jenkinsBuildUrl: matchedBuild.url,
},
run.version,
),
);
await this.auditService.record({
action: 'JENKINS_BUILD_LINKED',
resourceType: 'deploy_run',
resourceId: currentRun.id,
actorName: currentRun.operator,
after: {
queueId: matchedBuild.queueId,
buildNumber: matchedBuild.buildNumber,
buildUrl: matchedBuild.url,
source: 'queue_search',
},
});
return currentRun;
}
private async recordMatchedJenkinsBuild(
run: DeployRunSummary,
matchedBuild: JenkinsBuildParameterMatch,
): Promise<DeployRunSummary> {
if (matchedBuild.building || !matchedBuild.result) {
return this.recordJenkinsBuildStarted(run, {
buildNumber: matchedBuild.buildNumber,
buildUrl: matchedBuild.url,
source: 'parameter_search',
});
}
const currentRun = this.requireRun(
run.id,
await this.deployRunRepository.updateRun(
run.id,
{
jenkinsBuildNumber: matchedBuild.buildNumber,
jenkinsBuildUrl: matchedBuild.url,
},
run.version,
),
);
await this.auditService.record({
action: 'JENKINS_BUILD_LINKED',
resourceType: 'deploy_run',
resourceId: currentRun.id,
actorName: currentRun.operator,
after: {
buildNumber: matchedBuild.buildNumber,
buildUrl: matchedBuild.url,
source: 'parameter_search',
matchedParameterName: matchedBuild.matchedParameterName,
},
});
return currentRun;
}
private async findBuildByQueueId(
environment: ProjectEnvironmentSummary,
queueId: string,
): Promise<JenkinsBuildQueueMatch | undefined> {
return this.jenkinsClient.findBuildByQueueId(
environment.jenkinsJobPath,
queueId,
);
}
private async advanceJenkinsBuild(
run: DeployRunSummary,
project: ProjectSummary,
): Promise<{ run: DeployRunSummary; shouldContinue: boolean }> {
const environment = this.findEnvironment(project, run);
const health = this.jenkinsClient.healthSummary();
if (health.status === 'not_configured') {
const reason =
health.message ??
'Jenkins 尚未配置,已完成本地模拟构建。';
const currentRun = await this.completeStep(
run,
'jenkins-build',
'success',
);
await this.auditService.record({
action: 'JENKINS_DRY_RUN_COMPLETED',
resourceType: 'deploy_run',
resourceId: run.id,
actorName: run.operator,
after: {
mode: 'dry-run',
reason,
jobPath: environment.jenkinsJobPath,
},
});
return { run: currentRun, shouldContinue: true };
}
if (!environment.jenkinsJobPath) {
throw new AppError(
@@ -317,6 +483,8 @@ export class DeployExecutionService {
);
}
await this.assertJenkinsReadyForBuild(project, run, environment);
let currentRun = await this.markStep(run, 'jenkins-build', 'running');
const queueItem = await this.jenkinsClient.buildWithParameters(
environment.jenkinsJobPath,
@@ -361,6 +529,72 @@ export class DeployExecutionService {
return { run: currentRun, shouldContinue: false };
}
private async assertJenkinsReadyForBuild(
project: ProjectSummary,
run: DeployRunSummary,
environment: ProjectEnvironmentSummary,
): Promise<void> {
const health = this.jenkinsClient.healthSummary();
if (health.status === 'not_configured') {
throw new AppError(
'INTEGRATION_UNAVAILABLE',
`Jenkins 集成尚未配置,${health.message ?? '缺少必要配置'},无法触发构建`,
503,
{
integration: 'jenkins',
projectKey: project.key,
environment: run.environment,
jenkinsJobPath: environment.jenkinsJobPath,
},
);
}
try {
const info = await this.jenkinsClient.getJobInfo(
environment.jenkinsJobPath,
);
if (info.buildable === false) {
throw new AppError(
'INTEGRATION_REQUEST_FAILED',
`Jenkins job 当前不可构建:${environment.jenkinsJobPath}`,
502,
{
integration: 'jenkins',
projectKey: project.key,
environment: run.environment,
jenkinsJobPath: environment.jenkinsJobPath,
url: info.url,
},
);
}
} catch (error) {
if (
error instanceof IntegrationRequestError &&
error.details?.status === 404
) {
throw new AppError(
'INTEGRATION_REQUEST_FAILED',
`Jenkins job 不存在:${environment.jenkinsJobPath}`,
502,
{
integration: 'jenkins',
projectKey: project.key,
environment: run.environment,
jenkinsJobPath: environment.jenkinsJobPath,
},
);
}
if (error instanceof AppError) {
throw error;
}
throw error;
}
}
private async advanceNotification(
run: DeployRunSummary,
event: DeployNotificationEvent,
@@ -403,16 +637,8 @@ export class DeployExecutionService {
private async completeSuccessfulJenkinsRun(
run: DeployRunSummary,
): Promise<DeployRunSummary> {
let currentRun = await this.completeStep(
run,
'jenkins-build',
'success',
);
currentRun = await this.completeStep(
currentRun,
'health-check',
'success',
);
let currentRun = await this.completeStep(run, 'jenkins-build', 'success');
currentRun = await this.completeStep(currentRun, 'health-check', 'success');
currentRun = await this.advanceNotification(currentRun, 'succeeded');
currentRun = await this.completeStep(
currentRun,
@@ -613,14 +839,9 @@ export class DeployExecutionService {
run: DeployRunSummary | null,
): DeployRunSummary {
if (!run) {
throw new AppError(
'RESOURCE_NOT_FOUND',
`发布记录不存在:${id}`,
404,
{
id,
},
);
throw new AppError('RESOURCE_NOT_FOUND', `发布记录不存在:${id}`, 404, {
id,
});
}
return run;
+209 -1
View File
@@ -137,9 +137,46 @@ describe('DeployRunsService', () => {
requested: true as const,
}),
),
findBuildByQueueId: jest.fn(
(_jobPath: string, _queueId: string) => {
void _jobPath;
void _queueId;
return Promise.resolve(undefined);
},
),
findLatestBuildByParameter: jest.fn(
(
_jobPath: string,
_parameterName: string,
_parameterValue: string,
) => {
void _jobPath;
void _parameterName;
void _parameterValue;
return Promise.resolve(
undefined as Awaited<
ReturnType<JenkinsClient['findLatestBuildByParameter']>
>,
);
},
),
getProgressiveText: jest.fn(() =>
Promise.resolve({
text: 'build log',
nextStart: 9,
hasMore: false,
}),
),
} satisfies Pick<
JenkinsClient,
'healthSummary' | 'cancelQueueItem' | 'stopBuild'
| 'healthSummary'
| 'cancelQueueItem'
| 'stopBuild'
| 'findBuildByQueueId'
| 'findLatestBuildByParameter'
| 'getProgressiveText'
>;
const auth = {
listAllowedProjectKeys: jest.fn(() =>
@@ -327,6 +364,177 @@ describe('DeployRunsService', () => {
expect(execution.execute).toHaveBeenCalled();
});
it('syncs visible Jenkins runs before returning the run list', async () => {
const runningRun: DeployRunSummary = {
...baseRun,
id: 'run_running',
status: 'running',
jenkinsBuildNumber: 7,
};
const pendingRun: DeployRunSummary = {
...baseRun,
id: 'run_pending',
status: 'pending',
};
const { execution, service } = createService({
runs: [runningRun, pendingRun],
});
const runs = await service.listRuns();
expect(runs).toEqual([
expect.objectContaining({ id: 'run_running', status: 'success' }),
expect.objectContaining({ id: 'run_pending', status: 'pending' }),
]);
expect(execution.syncJenkinsRun).toHaveBeenCalledTimes(1);
expect(execution.syncJenkinsRun).toHaveBeenCalledWith(
runningRun,
expect.objectContaining({ key: runningRun.projectKey }),
);
});
it('syncs a Jenkins run before returning run detail', async () => {
const runningRun: DeployRunSummary = {
...baseRun,
id: 'run_running',
status: 'running',
jenkinsBuildNumber: 7,
};
const { execution, service } = createService({
runs: [runningRun],
});
const run = await service.getRun('run_running');
expect(run.status).toBe('success');
expect(execution.syncJenkinsRun).toHaveBeenCalledWith(
runningRun,
expect.objectContaining({ key: runningRun.projectKey }),
);
});
it('keeps returning stale read data when read-through Jenkins sync fails', async () => {
const runningRun: DeployRunSummary = {
...baseRun,
id: 'run_running',
status: 'running',
jenkinsBuildNumber: 7,
};
const { execution, service } = createService({
runs: [runningRun],
});
execution.syncJenkinsRun.mockRejectedValueOnce(
new Error('Jenkins unavailable'),
);
const run = await service.getRun('run_running');
expect(run).toBe(runningRun);
});
it('links a missing Jenkins build before reading logs', async () => {
const unlinkedRun: DeployRunSummary = {
...baseRun,
id: 'run_log_missing_build',
status: 'running',
};
const { audit, execution, jenkins, repository, service } = createService({
runs: [unlinkedRun],
});
const findLatestBuildByParameter =
jenkins.findLatestBuildByParameter as jest.MockedFunction<
JenkinsClient['findLatestBuildByParameter']
>;
findLatestBuildByParameter.mockResolvedValueOnce({
jobPath: 'my-project-dev/test-access-manage-develop',
buildNumber: 31,
building: false,
result: 'SUCCESS',
url: 'https://jenkins.example/job/31/',
matchedParameterName: 'DEPLOY_RUN_ID',
matchedParameterValue: 'run_log_missing_build',
});
const log = await service.getJenkinsLog('run_log_missing_build', undefined);
expect(log.buildNumber).toBe(31);
expect(log.text).toBe('build log');
expect(jenkins.findLatestBuildByParameter).toHaveBeenCalledWith(
'my-project-dev/test-access-manage-develop',
'DEPLOY_RUN_ID',
'run_log_missing_build',
);
expect(repository.updateRun).toHaveBeenCalledWith(
'run_log_missing_build',
expect.objectContaining({
jenkinsBuildNumber: 31,
jenkinsBuildUrl: 'https://jenkins.example/job/31/',
}),
unlinkedRun.version,
);
expect(execution.syncJenkinsRun).toHaveBeenCalledWith(
expect.objectContaining({
id: 'run_log_missing_build',
jenkinsBuildNumber: 31,
}),
expect.objectContaining({ key: 'access-manage' }),
);
expect(audit.record).toHaveBeenCalledWith(
expect.objectContaining({ action: 'JENKINS_BUILD_LINKED' }),
);
});
it('links a missing Jenkins build by queue id before reading logs', async () => {
const unlinkedRun: DeployRunSummary = {
...baseRun,
id: 'run_log_missing_queue_build',
status: 'running',
jenkinsQueueId: '431',
};
const { execution, jenkins, repository, service } = createService({
runs: [unlinkedRun],
});
const findBuildByQueueId =
jenkins.findBuildByQueueId as unknown as jest.MockedFunction<
JenkinsClient['findBuildByQueueId']
>;
findBuildByQueueId.mockResolvedValueOnce({
jobPath: 'my-project-dev/test-access-manage-develop',
buildNumber: 19,
building: false,
result: 'SUCCESS',
url: 'https://jenkins.example/job/19/',
queueId: '431',
});
const log = await service.getJenkinsLog(
'run_log_missing_queue_build',
undefined,
);
expect(log.buildNumber).toBe(19);
expect(jenkins.findBuildByQueueId).toHaveBeenCalledWith(
'my-project-dev/test-access-manage-develop',
'431',
);
expect(repository.updateRun).toHaveBeenCalledWith(
'run_log_missing_queue_build',
expect.objectContaining({
jenkinsQueueId: '431',
jenkinsBuildNumber: 19,
jenkinsBuildUrl: 'https://jenkins.example/job/19/',
}),
unlinkedRun.version,
);
expect(execution.syncJenkinsRun).toHaveBeenCalledWith(
expect.objectContaining({
id: 'run_log_missing_queue_build',
jenkinsBuildNumber: 19,
}),
expect.objectContaining({ key: 'access-manage' }),
);
});
it('cancels a queued Jenkins execution before marking the run canceled', async () => {
const queuedRun: DeployRunSummary = {
...baseRun,
+141 -32
View File
@@ -27,6 +27,10 @@ import {
DeployRunSummary,
} from './deploy-run.types';
type ReadRunsOptions = {
syncJenkins?: boolean;
};
const escapeControl = String.fromCharCode(27);
const jenkinsConsoleNotePattern = new RegExp(
`${escapeControl}\\[8mha:/{4}[A-Za-z0-9+/=]+${escapeControl}\\[0m`,
@@ -49,35 +53,41 @@ export class DeployRunsService {
private readonly authService: AuthService,
) {}
async listRuns(user?: AuthenticatedUser): Promise<DeployRunSummary[]> {
async listRuns(
user?: AuthenticatedUser,
options: ReadRunsOptions = {},
): Promise<DeployRunSummary[]> {
const runs = await this.deployRunRepository.findMany();
const shouldSyncJenkins = options.syncJenkins ?? true;
if (!user || user.role === 'super_admin') {
return runs;
return shouldSyncJenkins ? this.syncJenkinsCandidatesForRead(runs) : runs;
}
const allowedProjectKeys = new Set(
await this.authService.listAllowedProjectKeys(user),
);
return runs.filter((run) => allowedProjectKeys.has(run.projectKey));
const visibleRuns = runs.filter((run) =>
allowedProjectKeys.has(run.projectKey),
);
return shouldSyncJenkins
? this.syncJenkinsCandidatesForRead(visibleRuns)
: visibleRuns;
}
async getRun(
id: string,
user?: AuthenticatedUser,
options: ReadRunsOptions = {},
): Promise<DeployRunSummary> {
const run = await this.deployRunRepository.findById(id);
if (!run) {
throw new AppError(
'RESOURCE_NOT_FOUND',
`发布记录不存在:${id}`,
404,
{
id,
},
);
throw new AppError('RESOURCE_NOT_FOUND', `发布记录不存在:${id}`, 404, {
id,
});
}
if (user) {
@@ -88,7 +98,9 @@ export class DeployRunsService {
);
}
return run;
return (options.syncJenkins ?? true)
? this.syncJenkinsCandidateForRead(run)
: run;
}
async createRun(
@@ -144,7 +156,7 @@ export class DeployRunsService {
id: string,
user?: AuthenticatedUser,
): Promise<DeployRunSummary> {
const source = await this.getRun(id, user);
const source = await this.getRun(id, user, { syncJenkins: false });
if (user) {
await this.authService.requireProjectPermission(
@@ -211,7 +223,7 @@ export class DeployRunsService {
id: string,
user?: AuthenticatedUser,
): Promise<DeployRunSummary> {
const source = await this.getRun(id, user);
const source = await this.getRun(id, user, { syncJenkins: false });
if (!['failed', 'canceled'].includes(source.status)) {
throw new AppError(
@@ -250,7 +262,7 @@ export class DeployRunsService {
id: string,
user?: AuthenticatedUser,
): Promise<DeployRunSummary> {
const run = await this.getRun(id, user);
const run = await this.getRun(id, user, { syncJenkins: false });
if (user) {
await this.authService.requireProjectPermission(
@@ -270,9 +282,19 @@ export class DeployRunsService {
start: string | undefined,
user?: AuthenticatedUser,
): Promise<DeployRunJenkinsLog> {
const run = await this.getRun(id, user);
let run = await this.getRun(id, user);
if (run.jenkinsBuildNumber === undefined || run.jenkinsBuildNumber === null) {
if (
run.jenkinsBuildNumber === undefined ||
run.jenkinsBuildNumber === null
) {
run = await this.linkMissingJenkinsBuildForLog(run);
}
if (
run.jenkinsBuildNumber === undefined ||
run.jenkinsBuildNumber === null
) {
throw new AppError(
'VALIDATION_FAILED',
`发布记录 ${id} 尚未关联 Jenkins build,无法读取日志`,
@@ -331,7 +353,7 @@ export class DeployRunsService {
this.authService.assertSuperAdmin(user, '无权限同步全部 Jenkins 状态');
}
const runs = await this.listRuns();
const runs = await this.listRuns(undefined, { syncJenkins: false });
const candidates = runs.filter((run) => this.isJenkinsSyncCandidate(run));
const syncedRuns: DeployRunSummary[] = [];
const failedRuns: DeployRunJenkinsSyncSummary['failedRuns'] = [];
@@ -399,6 +421,102 @@ export class DeployRunsService {
);
}
private async syncJenkinsCandidatesForRead(
runs: DeployRunSummary[],
): Promise<DeployRunSummary[]> {
const result: DeployRunSummary[] = [];
for (const run of runs) {
result.push(await this.syncJenkinsCandidateForRead(run));
}
return result;
}
private async syncJenkinsCandidateForRead(
run: DeployRunSummary,
): Promise<DeployRunSummary> {
if (!this.isJenkinsSyncCandidate(run)) {
return run;
}
try {
const project = await this.projectsService.getProject(run.projectKey);
return await this.deployExecutionService.syncJenkinsRun(run, project);
} catch {
return run;
}
}
private async linkMissingJenkinsBuildForLog(
run: DeployRunSummary,
): Promise<DeployRunSummary> {
try {
const project = await this.projectsService.getProject(run.projectKey);
const environment = this.findEnvironment(project, run);
const matchedBuild = run.jenkinsQueueId
? await this.jenkinsClient.findBuildByQueueId(
environment.jenkinsJobPath,
run.jenkinsQueueId,
)
: await this.jenkinsClient.findLatestBuildByParameter(
environment.jenkinsJobPath,
'DEPLOY_RUN_ID',
run.id,
);
if (!matchedBuild) {
return run;
}
const linkedRun = this.requireRun(
run.id,
await this.deployRunRepository.updateRun(
run.id,
{
status:
matchedBuild.building || !matchedBuild.result
? 'running'
: run.status,
jenkinsQueueId:
'queueId' in matchedBuild ? matchedBuild.queueId : run.jenkinsQueueId,
jenkinsBuildNumber: matchedBuild.buildNumber,
jenkinsBuildUrl: matchedBuild.url,
},
run.version,
),
);
await this.auditService.record({
action: 'JENKINS_BUILD_LINKED',
resourceType: 'deploy_run',
resourceId: linkedRun.id,
actorName: linkedRun.operator,
after: {
queueId: 'queueId' in matchedBuild ? matchedBuild.queueId : undefined,
buildNumber: matchedBuild.buildNumber,
buildUrl: matchedBuild.url,
source:
'queueId' in matchedBuild
? 'log_queue_search'
: 'log_parameter_search',
matchedParameterName:
'matchedParameterName' in matchedBuild
? matchedBuild.matchedParameterName
: undefined,
},
});
return await this.deployExecutionService.syncJenkinsRun(
linkedRun,
project,
);
} catch {
return run;
}
}
private normalizeLogStart(start: string | undefined): number {
if (start === undefined || start === '') {
return 0;
@@ -539,10 +657,7 @@ export class DeployRunsService {
}
if (!new RegExp(environment.refPattern).test(input.ref)) {
await this.recordDeployRejected(
input,
'Ref 不符合发布策略',
);
await this.recordDeployRejected(input, 'Ref 不符合发布策略');
throw new AppError(
'VALIDATION_FAILED',
`Ref ${input.ref} 不符合 ${environment.releasePolicy} 发布策略`,
@@ -593,8 +708,7 @@ export class DeployRunsService {
}
private toSafeSyncError(error: unknown): string {
const message =
error instanceof Error ? error.message : 'Jenkins 同步失败';
const message = error instanceof Error ? error.message : 'Jenkins 同步失败';
return redactSensitive({ message }).message;
}
@@ -604,14 +718,9 @@ export class DeployRunsService {
run: DeployRunSummary | null,
): DeployRunSummary {
if (!run) {
throw new AppError(
'RESOURCE_NOT_FOUND',
`发布记录不存在:${id}`,
404,
{
id,
},
);
throw new AppError('RESOURCE_NOT_FOUND', `发布记录不存在:${id}`, 404, {
id,
});
}
return run;
@@ -0,0 +1,184 @@
import { ConfigService } from '@nestjs/config';
import { EnvConfig } from '../../config/env.schema';
import { JenkinsClient } from './jenkins.client';
describe('JenkinsClient', () => {
afterEach(() => {
jest.restoreAllMocks();
});
it('retries build requests with a Jenkins crumb when CSRF protection rejects the first POST', async () => {
const client = createClient();
const fetchMock = jest
.spyOn(global, 'fetch')
.mockImplementation(
(input: string | URL | Request, init?: RequestInit) => {
const url =
typeof input === 'string'
? input
: input instanceof URL
? input.toString()
: input.url;
if (url.endsWith('/crumbIssuer/api/json')) {
return Promise.resolve(
new Response(
JSON.stringify({
crumbRequestField: 'Jenkins-Crumb',
crumb: 'crumb-001',
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
),
);
}
const headers = new Headers(init?.headers);
if (!headers.has('Jenkins-Crumb')) {
return Promise.resolve(
new Response('No valid crumb', { status: 403 }),
);
}
return Promise.resolve(
new Response('', {
status: 201,
headers: {
location: 'https://jenkins.example/queue/item/42/',
},
}),
);
},
);
const queueItem = await client.buildWithParameters(
'my-project-dev/test-role-user-develop',
{
GIT_REF: 'develop',
},
);
expect(queueItem.queueId).toBe('42');
expect(fetchMock).toHaveBeenCalledTimes(3);
const retryHeaders = new Headers(
(fetchMock.mock.calls[2][1] as RequestInit).headers,
);
expect(retryHeaders.get('Jenkins-Crumb')).toBe('crumb-001');
});
it('fails build requests when Jenkins does not return a queue location', async () => {
const client = createClient();
jest.spyOn(global, 'fetch').mockResolvedValue(
new Response('', {
status: 201,
}),
);
await expect(
client.buildWithParameters('my-project-dev/test-role-user-develop', {
GIT_REF: 'develop',
}),
).rejects.toMatchObject({
code: 'INTEGRATION_REQUEST_FAILED',
message: 'Jenkins 未返回队列地址',
});
});
it('finds the latest build by Jenkins parameter value', async () => {
const client = createClient();
jest.spyOn(global, 'fetch').mockResolvedValue(
new Response(
JSON.stringify({
builds: [
{
number: 21,
url: 'https://jenkins.example/job/my-project-dev/job/test-access-manage-develop/21/',
building: false,
result: 'SUCCESS',
actions: [
{
parameters: [
{
name: 'DEPLOY_RUN_ID',
value: 'run_001',
},
],
},
],
},
],
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
),
);
const build = await client.findLatestBuildByParameter(
'my-project-dev/test-access-manage-develop',
'DEPLOY_RUN_ID',
'run_001',
);
expect(build).toEqual(
expect.objectContaining({
buildNumber: 21,
result: 'SUCCESS',
matchedParameterName: 'DEPLOY_RUN_ID',
}),
);
});
it('finds a build by Jenkins queue id', async () => {
const client = createClient();
jest.spyOn(global, 'fetch').mockResolvedValue(
new Response(
JSON.stringify({
builds: [
{
number: 19,
queueId: 431,
url: 'https://jenkins.example/job/my-project-dev/job/test-access-manage-develop/19/',
building: false,
result: 'SUCCESS',
},
],
}),
{
status: 200,
headers: { 'content-type': 'application/json' },
},
),
);
const build = await client.findBuildByQueueId(
'my-project-dev/test-access-manage-develop',
'431',
);
expect(build).toEqual(
expect.objectContaining({
buildNumber: 19,
queueId: '431',
result: 'SUCCESS',
}),
);
});
});
function createClient(): JenkinsClient {
const values: Partial<Record<keyof EnvConfig, string>> = {
JENKINS_BASE_URL: 'https://jenkins.example',
JENKINS_USERNAME: 'devops-bot',
JENKINS_API_TOKEN: 'secret-token',
};
const config = {
get: jest.fn((key: keyof EnvConfig) => values[key]),
} as unknown as ConfigService<EnvConfig, true>;
return new JenkinsClient(config);
}
+247 -15
View File
@@ -10,6 +10,8 @@ import {
import { redactSensitive } from '../../common/security/redact-sensitive';
import { EnvConfig } from '../../config/env.schema';
import {
JenkinsBuildParameterMatch,
JenkinsBuildQueueMatch,
JenkinsBuildStatus,
JenkinsCancelResult,
JenkinsJobInfo,
@@ -17,8 +19,21 @@ import {
JenkinsQueueItem,
} from './jenkins.types';
type JenkinsConfig = {
baseUrl: string;
username: string;
apiToken: string;
};
type JenkinsCrumb = {
headerName: string;
value: string;
};
@Injectable()
export class JenkinsClient {
private crumb: JenkinsCrumb | undefined;
constructor(private readonly config: ConfigService<EnvConfig, true>) {}
healthSummary(): { status: 'ok' | 'not_configured'; message?: string } {
@@ -62,7 +77,7 @@ export class JenkinsClient {
});
}
return this.toQueueItem(response);
return this.toQueueItem(path, response);
}
private async buildWithoutParameters(
@@ -73,13 +88,22 @@ export class JenkinsClient {
method: 'POST',
});
return this.toQueueItem(response);
return this.toQueueItem(path, response);
}
private toQueueItem(response: Response): JenkinsQueueItem {
private toQueueItem(path: string, response: Response): JenkinsQueueItem {
const location = response.headers.get('location') ?? undefined;
const queueId = location?.split('/').filter(Boolean).pop();
if (!queueId) {
throw new IntegrationRequestError('jenkins', 'Jenkins 未返回队列地址', {
path,
status: response.status,
});
}
return {
queueId: location?.split('/').filter(Boolean).pop() ?? 'unknown',
queueId,
executableUrl: location,
};
}
@@ -122,6 +146,115 @@ export class JenkinsClient {
};
}
async findBuildByQueueId(
jobPath: string,
queueId: string,
): Promise<JenkinsBuildQueueMatch | undefined> {
this.assertConfigured();
const tree = 'builds[number,url,building,result,queueId]';
const response = await this.request(
`${this.jobPath(jobPath)}/api/json?tree=${encodeURIComponent(tree)}`,
);
const payload = (await response.json()) as {
builds?: Array<{
number?: number;
queueId?: number | string;
url?: string;
building?: boolean;
result?: JenkinsBuildStatus['result'];
}>;
};
const build = (payload.builds ?? []).find(
(item) =>
typeof item.number === 'number' && String(item.queueId) === queueId,
);
if (!build || typeof build.number !== 'number') {
return undefined;
}
return {
jobPath,
buildNumber: build.number,
building: Boolean(build.building),
result: build.result,
url: build.url,
queueId,
};
}
async findLatestBuildByParameter(
jobPath: string,
parameterName: string,
parameterValue: string,
): Promise<JenkinsBuildParameterMatch | undefined> {
this.assertConfigured();
const tree =
'builds[number,url,building,result,actions[parameters[name,value]]]';
const response = await this.request(
`${this.jobPath(jobPath)}/api/json?tree=${encodeURIComponent(tree)}`,
);
const payload = (await response.json()) as {
builds?: Array<{
number?: number;
url?: string;
building?: boolean;
result?: JenkinsBuildStatus['result'];
actions?: Array<{
parameters?: Array<{ name?: string; value?: unknown }>;
}>;
}>;
};
for (const build of payload.builds ?? []) {
if (typeof build.number !== 'number') {
continue;
}
const parameter = build.actions
?.flatMap((action) => action.parameters ?? [])
.find(
(item) =>
item.name === parameterName &&
this.jenkinsParameterValueToString(item.value) === parameterValue,
);
if (!parameter) {
continue;
}
return {
jobPath,
buildNumber: build.number,
building: Boolean(build.building),
result: build.result,
url: build.url,
matchedParameterName: parameterName,
matchedParameterValue: parameterValue,
};
}
return undefined;
}
private jenkinsParameterValueToString(value: unknown): string {
if (value === undefined || value === null) {
return '';
}
if (
typeof value === 'string' ||
typeof value === 'number' ||
typeof value === 'boolean' ||
typeof value === 'bigint'
) {
return String(value);
}
return JSON.stringify(value) ?? '';
}
async getJobInfo(jobPath: string): Promise<JenkinsJobInfo> {
this.assertConfigured();
const response = await this.request(
@@ -191,17 +324,17 @@ export class JenkinsClient {
init: RequestInit = {},
options: { allowFailure?: boolean } = {},
): Promise<Response> {
const baseUrl = this.config.get('JENKINS_BASE_URL', { infer: true }) ?? '';
const username = this.config.get('JENKINS_USERNAME', { infer: true }) ?? '';
const apiToken =
this.config.get('JENKINS_API_TOKEN', { infer: true }) ?? '';
const response = await fetch(`${baseUrl}${path}`, {
...init,
headers: {
authorization: `Basic ${Buffer.from(`${username}:${apiToken}`).toString('base64')}`,
...(init.headers ?? {}),
},
});
const jenkinsConfig = this.readConfig();
let response = await this.sendRequest(jenkinsConfig, path, init);
if (await this.shouldRetryWithCrumb(response, init)) {
this.crumb = undefined;
const crumb = await this.fetchCrumb(jenkinsConfig);
if (crumb) {
response = await this.sendRequest(jenkinsConfig, path, init, crumb);
}
}
if (!response.ok && !options.allowFailure) {
throw this.requestError(path, {
@@ -213,6 +346,105 @@ export class JenkinsClient {
return response;
}
private async sendRequest(
config: JenkinsConfig,
path: string,
init: RequestInit,
crumb?: JenkinsCrumb,
): Promise<Response> {
return fetch(`${config.baseUrl}${path}`, {
...init,
headers: {
authorization: this.authorizationHeader(config),
...(crumb ? { [crumb.headerName]: crumb.value } : {}),
...this.normalizeHeaders(init.headers),
},
});
}
private async shouldRetryWithCrumb(
response: Response,
init: RequestInit,
): Promise<boolean> {
if ((init.method ?? 'GET').toUpperCase() !== 'POST') {
return false;
}
if (response.status !== 403) {
return false;
}
const responseText = await response
.clone()
.text()
.catch(() => '');
return /(crumb|csrf|No valid crumb)/i.test(responseText);
}
private async fetchCrumb(
config: JenkinsConfig,
): Promise<JenkinsCrumb | undefined> {
if (this.crumb) {
return this.crumb;
}
const response = await fetch(`${config.baseUrl}/crumbIssuer/api/json`, {
headers: {
authorization: this.authorizationHeader(config),
},
}).catch(() => undefined);
if (!response?.ok) {
return undefined;
}
const payload = (await response.json().catch(() => undefined)) as
| {
crumbRequestField?: string;
crumb?: string;
}
| undefined;
if (!payload?.crumbRequestField || !payload.crumb) {
return undefined;
}
this.crumb = {
headerName: payload.crumbRequestField,
value: payload.crumb,
};
return this.crumb;
}
private readConfig(): JenkinsConfig {
return {
baseUrl: this.config.get('JENKINS_BASE_URL', { infer: true }) ?? '',
username: this.config.get('JENKINS_USERNAME', { infer: true }) ?? '',
apiToken: this.config.get('JENKINS_API_TOKEN', { infer: true }) ?? '',
};
}
private authorizationHeader(config: JenkinsConfig): string {
return `Basic ${Buffer.from(`${config.username}:${config.apiToken}`).toString('base64')}`;
}
private normalizeHeaders(
headers: HeadersInit | undefined,
): Record<string, string> {
if (!headers) {
return {};
}
const normalized: Record<string, string> = {};
new Headers(headers).forEach((value, key) => {
normalized[key] = value;
});
return normalized;
}
private jobPath(jobPath: string, action?: string): string {
const encoded = `/job/${jobPath.split('/').map(encodeURIComponent).join('/job/')}`;
@@ -12,6 +12,15 @@ export type JenkinsBuildStatus = {
url?: string;
};
export type JenkinsBuildParameterMatch = JenkinsBuildStatus & {
matchedParameterName: string;
matchedParameterValue: string;
};
export type JenkinsBuildQueueMatch = JenkinsBuildStatus & {
queueId: string;
};
export type JenkinsJobInfo = {
jobPath: string;
name?: string;