湛兮
542 lines
15 KiB
TypeScript
542 lines
15 KiB
TypeScript
import { Injectable } from '@nestjs/common';
|
|
import { AuditService } from '../audit/audit.service';
|
|
import { AppError } from '../common/errors/app-error';
|
|
import { redactSensitive } from '../common/security/redact-sensitive';
|
|
import { JenkinsClient } from '../integrations/jenkins/jenkins.client';
|
|
import { JenkinsBuildStatus } from '../integrations/jenkins/jenkins.types';
|
|
import {
|
|
DeployNotificationEvent,
|
|
DeployNotificationService,
|
|
} from '../notifications/wecom/deploy-notification.service';
|
|
import {
|
|
ProjectEnvironmentSummary,
|
|
ProjectSummary,
|
|
} from '../projects/project.types';
|
|
import { DeployRunRepository } from './deploy-run.repository';
|
|
import { DeployRunStepStatus, DeployRunSummary } from './deploy-run.types';
|
|
|
|
@Injectable()
|
|
export class DeployExecutionService {
|
|
constructor(
|
|
private readonly deployRunRepository: DeployRunRepository,
|
|
private readonly auditService: AuditService,
|
|
private readonly deployNotificationService: DeployNotificationService,
|
|
private readonly jenkinsClient: JenkinsClient,
|
|
) {}
|
|
|
|
async execute(
|
|
run: DeployRunSummary,
|
|
project: ProjectSummary,
|
|
): Promise<DeployRunSummary> {
|
|
if (run.status !== 'pending') {
|
|
return run;
|
|
}
|
|
|
|
let currentRun = this.requireRun(
|
|
run.id,
|
|
this.deployRunRepository.updateMemoryRun(run.id, { status: 'running' }),
|
|
);
|
|
let currentStep: string | undefined;
|
|
|
|
this.auditService.record({
|
|
action: 'DEPLOY_EXECUTION_STARTED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
projectKey: currentRun.projectKey,
|
|
environment: currentRun.environment,
|
|
ref: currentRun.ref,
|
|
status: currentRun.status,
|
|
},
|
|
});
|
|
|
|
try {
|
|
currentStep = 'gitea-check';
|
|
currentRun = this.completeStep(currentRun.id, currentStep, 'success');
|
|
|
|
currentStep = 'jenkins-build';
|
|
const jenkinsResult = await this.advanceJenkinsBuild(currentRun, project);
|
|
currentRun = jenkinsResult.run;
|
|
|
|
if (!jenkinsResult.shouldContinue) {
|
|
await this.recordNotificationResult(currentRun, 'queued');
|
|
return currentRun;
|
|
}
|
|
|
|
currentStep = 'health-check';
|
|
currentRun = this.completeStep(currentRun.id, currentStep, 'success');
|
|
|
|
currentStep = 'notify-wecom';
|
|
currentRun = await this.advanceNotification(currentRun, 'succeeded');
|
|
|
|
currentStep = 'finish-release';
|
|
currentRun = this.completeStep(currentRun.id, currentStep, 'success');
|
|
currentRun = this.requireRun(
|
|
currentRun.id,
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'success',
|
|
}),
|
|
);
|
|
|
|
this.auditService.record({
|
|
action: 'DEPLOY_SUCCEEDED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
status: currentRun.status,
|
|
jenkinsQueueId: currentRun.jenkinsQueueId,
|
|
jenkinsBuildNumber: currentRun.jenkinsBuildNumber,
|
|
},
|
|
});
|
|
|
|
return currentRun;
|
|
} catch (error) {
|
|
const failureSummary = this.toFailureSummary(error);
|
|
|
|
if (currentStep) {
|
|
currentRun =
|
|
this.deployRunRepository.updateMemoryStep(
|
|
currentRun.id,
|
|
currentStep,
|
|
{
|
|
status: 'failed',
|
|
errorSummary: failureSummary,
|
|
},
|
|
) ?? currentRun;
|
|
}
|
|
|
|
currentRun =
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'failed',
|
|
failureSummary,
|
|
}) ?? currentRun;
|
|
|
|
currentRun = await this.advanceNotification(currentRun, 'failed');
|
|
|
|
this.auditService.record({
|
|
action: 'DEPLOY_FAILED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
status: currentRun.status,
|
|
failedStep: currentStep,
|
|
failureSummary,
|
|
},
|
|
});
|
|
|
|
return currentRun;
|
|
}
|
|
}
|
|
|
|
async syncJenkinsRun(
|
|
run: DeployRunSummary,
|
|
project: ProjectSummary,
|
|
): Promise<DeployRunSummary> {
|
|
if (['success', 'failed', 'canceled'].includes(run.status)) {
|
|
return run;
|
|
}
|
|
|
|
if (!run.jenkinsQueueId && !run.jenkinsBuildNumber) {
|
|
return run;
|
|
}
|
|
|
|
const environment = this.findEnvironment(project, run);
|
|
let currentRun = run;
|
|
let buildNumber = currentRun.jenkinsBuildNumber;
|
|
|
|
if (!buildNumber && currentRun.jenkinsQueueId) {
|
|
const queueItem = await this.jenkinsClient.getQueueItem(
|
|
currentRun.jenkinsQueueId,
|
|
);
|
|
|
|
if (!queueItem.buildNumber) {
|
|
return this.requireRun(
|
|
currentRun.id,
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'queued',
|
|
jenkinsQueueId: queueItem.queueId,
|
|
jenkinsBuildUrl: queueItem.executableUrl,
|
|
}),
|
|
);
|
|
}
|
|
|
|
buildNumber = queueItem.buildNumber;
|
|
currentRun = this.requireRun(
|
|
currentRun.id,
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'running',
|
|
jenkinsQueueId: queueItem.queueId,
|
|
jenkinsBuildNumber: queueItem.buildNumber,
|
|
jenkinsBuildUrl: queueItem.executableUrl,
|
|
}),
|
|
);
|
|
currentRun = this.markStep(
|
|
currentRun.id,
|
|
'jenkins-build',
|
|
'running',
|
|
`Jenkins build #${queueItem.buildNumber} is running.`,
|
|
);
|
|
|
|
this.auditService.record({
|
|
action: 'JENKINS_BUILD_STARTED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
queueId: queueItem.queueId,
|
|
buildNumber: queueItem.buildNumber,
|
|
buildUrl: queueItem.executableUrl,
|
|
},
|
|
});
|
|
}
|
|
|
|
if (!buildNumber) {
|
|
return currentRun;
|
|
}
|
|
|
|
const buildStatus = await this.jenkinsClient.getBuildStatus(
|
|
environment.jenkinsJobPath,
|
|
buildNumber,
|
|
);
|
|
|
|
currentRun = this.requireRun(
|
|
currentRun.id,
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: buildStatus.building || !buildStatus.result ? 'running' : currentRun.status,
|
|
jenkinsBuildNumber: buildStatus.buildNumber,
|
|
jenkinsBuildUrl: buildStatus.url,
|
|
}),
|
|
);
|
|
|
|
if (buildStatus.building || !buildStatus.result) {
|
|
return this.markStep(
|
|
currentRun.id,
|
|
'jenkins-build',
|
|
'running',
|
|
`Jenkins build #${buildStatus.buildNumber} is running.`,
|
|
);
|
|
}
|
|
|
|
if (buildStatus.result === 'SUCCESS') {
|
|
return this.completeSuccessfulJenkinsRun(currentRun);
|
|
}
|
|
|
|
return this.failJenkinsRun(
|
|
currentRun,
|
|
buildStatus.result,
|
|
environment.jenkinsJobPath,
|
|
);
|
|
}
|
|
|
|
private async advanceJenkinsBuild(
|
|
run: DeployRunSummary,
|
|
project: ProjectSummary,
|
|
): Promise<{ run: DeployRunSummary; shouldContinue: boolean }> {
|
|
const environment = this.findEnvironment(project, run);
|
|
const health = this.jenkinsClient.healthSummary();
|
|
|
|
if (health.status === 'not_configured') {
|
|
const reason =
|
|
health.message ??
|
|
'Jenkins is not configured; simulated build completed.';
|
|
const currentRun = this.completeStep(run.id, 'jenkins-build', 'success');
|
|
|
|
this.auditService.record({
|
|
action: 'JENKINS_DRY_RUN_COMPLETED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: run.id,
|
|
actorName: run.operator,
|
|
after: {
|
|
mode: 'dry-run',
|
|
reason,
|
|
jobPath: environment.jenkinsJobPath,
|
|
},
|
|
});
|
|
|
|
return { run: currentRun, shouldContinue: true };
|
|
}
|
|
|
|
if (!environment.jenkinsJobPath) {
|
|
throw new AppError(
|
|
'VALIDATION_FAILED',
|
|
`Project ${project.key} ${run.environment} Jenkins job path is empty`,
|
|
400,
|
|
{ projectKey: project.key, environment: run.environment },
|
|
);
|
|
}
|
|
|
|
this.markStep(run.id, 'jenkins-build', 'running');
|
|
const queueItem = await this.jenkinsClient.buildWithParameters(
|
|
environment.jenkinsJobPath,
|
|
this.buildParameters(run),
|
|
);
|
|
const nextStatus = queueItem.buildNumber ? 'running' : 'queued';
|
|
let currentRun = this.requireRun(
|
|
run.id,
|
|
this.deployRunRepository.updateMemoryRun(run.id, {
|
|
status: nextStatus,
|
|
jenkinsQueueId: queueItem.queueId,
|
|
jenkinsBuildNumber: queueItem.buildNumber,
|
|
jenkinsBuildUrl: queueItem.executableUrl,
|
|
}),
|
|
);
|
|
currentRun = this.markStep(
|
|
run.id,
|
|
'jenkins-build',
|
|
nextStatus,
|
|
queueItem.buildNumber
|
|
? `Jenkins build #${queueItem.buildNumber} is running.`
|
|
: `Jenkins queue ${queueItem.queueId} accepted.`,
|
|
);
|
|
|
|
this.auditService.record({
|
|
action: 'JENKINS_BUILD_TRIGGERED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: run.id,
|
|
actorName: run.operator,
|
|
after: {
|
|
jobPath: environment.jenkinsJobPath,
|
|
queueId: queueItem.queueId,
|
|
buildNumber: queueItem.buildNumber,
|
|
},
|
|
parameterDigest: this.buildParameters(run),
|
|
});
|
|
|
|
return { run: currentRun, shouldContinue: false };
|
|
}
|
|
|
|
private async advanceNotification(
|
|
run: DeployRunSummary,
|
|
event: DeployNotificationEvent,
|
|
): Promise<DeployRunSummary> {
|
|
this.markStep(run.id, 'notify-wecom', 'running');
|
|
const notification =
|
|
await this.deployNotificationService.sendDeployRunEvent(run, event);
|
|
const stepStatus =
|
|
notification.status === 'sent'
|
|
? 'success'
|
|
: notification.status === 'failed'
|
|
? 'failed'
|
|
: 'skipped';
|
|
const currentRun = this.completeStep(
|
|
run.id,
|
|
'notify-wecom',
|
|
stepStatus,
|
|
notification.status === 'sent' ? undefined : notification.reason,
|
|
);
|
|
|
|
this.auditService.record({
|
|
action: `WECOM_NOTIFICATION_${notification.status.toUpperCase()}`,
|
|
resourceType: 'deploy_run',
|
|
resourceId: run.id,
|
|
actorName: run.operator,
|
|
after: {
|
|
event,
|
|
channel: notification.channel,
|
|
reason: notification.reason,
|
|
},
|
|
});
|
|
|
|
return currentRun;
|
|
}
|
|
|
|
private async completeSuccessfulJenkinsRun(
|
|
run: DeployRunSummary,
|
|
): Promise<DeployRunSummary> {
|
|
let currentRun = this.completeStep(run.id, 'jenkins-build', 'success');
|
|
currentRun = this.completeStep(currentRun.id, 'health-check', 'success');
|
|
currentRun = await this.advanceNotification(currentRun, 'succeeded');
|
|
currentRun = this.completeStep(currentRun.id, 'finish-release', 'success');
|
|
currentRun = this.requireRun(
|
|
currentRun.id,
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'success',
|
|
}),
|
|
);
|
|
|
|
this.auditService.record({
|
|
action: 'DEPLOY_SUCCEEDED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
status: currentRun.status,
|
|
jenkinsQueueId: currentRun.jenkinsQueueId,
|
|
jenkinsBuildNumber: currentRun.jenkinsBuildNumber,
|
|
},
|
|
});
|
|
|
|
return currentRun;
|
|
}
|
|
|
|
private async failJenkinsRun(
|
|
run: DeployRunSummary,
|
|
result: Exclude<NonNullable<JenkinsBuildStatus['result']>, 'SUCCESS'>,
|
|
jobPath: string,
|
|
): Promise<DeployRunSummary> {
|
|
const logExcerpt = await this.readFailureLogExcerpt(run, jobPath);
|
|
const failureSummary = `Jenkins build #${run.jenkinsBuildNumber ?? '-'} ${result}`;
|
|
let currentRun =
|
|
this.deployRunRepository.updateMemoryStep(run.id, 'jenkins-build', {
|
|
status: 'failed',
|
|
errorSummary: failureSummary,
|
|
logExcerpt,
|
|
}) ?? run;
|
|
currentRun =
|
|
this.deployRunRepository.updateMemoryRun(currentRun.id, {
|
|
status: 'failed',
|
|
failureSummary,
|
|
}) ?? currentRun;
|
|
currentRun = await this.advanceNotification(currentRun, 'failed');
|
|
|
|
this.auditService.record({
|
|
action: 'DEPLOY_FAILED',
|
|
resourceType: 'deploy_run',
|
|
resourceId: currentRun.id,
|
|
actorName: currentRun.operator,
|
|
after: {
|
|
status: currentRun.status,
|
|
failedStep: 'jenkins-build',
|
|
failureSummary,
|
|
},
|
|
});
|
|
|
|
return currentRun;
|
|
}
|
|
|
|
private async readFailureLogExcerpt(
|
|
run: DeployRunSummary,
|
|
jobPath: string,
|
|
): Promise<string | undefined> {
|
|
if (!run.jenkinsBuildNumber) {
|
|
return undefined;
|
|
}
|
|
|
|
try {
|
|
const log = await this.jenkinsClient.getProgressiveText(
|
|
jobPath,
|
|
run.jenkinsBuildNumber,
|
|
);
|
|
const logExcerpt = log.text
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.slice(-24)
|
|
.join('\n')
|
|
.slice(-4000);
|
|
|
|
return redactSensitive({ logExcerpt }).logExcerpt;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
private findEnvironment(
|
|
project: ProjectSummary,
|
|
run: DeployRunSummary,
|
|
): ProjectEnvironmentSummary {
|
|
const environment = project.environments.find(
|
|
(item) => item.name === run.environment,
|
|
);
|
|
|
|
if (!environment) {
|
|
throw new AppError(
|
|
'VALIDATION_FAILED',
|
|
`Project ${project.key} does not expose ${run.environment} environment`,
|
|
400,
|
|
{ projectKey: project.key, environment: run.environment },
|
|
);
|
|
}
|
|
|
|
return environment;
|
|
}
|
|
|
|
private buildParameters(run: DeployRunSummary): Record<string, string> {
|
|
return {
|
|
DEPLOY_RUN_ID: run.id,
|
|
PROJECT_KEY: run.projectKey,
|
|
DEPLOY_ENVIRONMENT: run.environment,
|
|
GIT_REF: run.ref,
|
|
OPERATOR: run.operator,
|
|
};
|
|
}
|
|
|
|
private completeStep(
|
|
runId: string,
|
|
bpmnNodeId: string,
|
|
status: Extract<DeployRunStepStatus, 'success' | 'skipped' | 'failed'>,
|
|
errorSummary?: string,
|
|
): DeployRunSummary {
|
|
return this.requireRun(
|
|
runId,
|
|
this.deployRunRepository.updateMemoryStep(runId, bpmnNodeId, {
|
|
status,
|
|
errorSummary,
|
|
}),
|
|
);
|
|
}
|
|
|
|
private markStep(
|
|
runId: string,
|
|
bpmnNodeId: string,
|
|
status: Extract<DeployRunStepStatus, 'queued' | 'running'>,
|
|
message?: string,
|
|
): DeployRunSummary {
|
|
return this.requireRun(
|
|
runId,
|
|
this.deployRunRepository.updateMemoryStep(runId, bpmnNodeId, {
|
|
status,
|
|
message,
|
|
}),
|
|
);
|
|
}
|
|
|
|
private async recordNotificationResult(
|
|
run: DeployRunSummary,
|
|
event: DeployNotificationEvent,
|
|
): Promise<void> {
|
|
const notification =
|
|
await this.deployNotificationService.sendDeployRunEvent(run, event);
|
|
|
|
this.auditService.record({
|
|
action: `WECOM_NOTIFICATION_${notification.status.toUpperCase()}`,
|
|
resourceType: 'deploy_run',
|
|
resourceId: run.id,
|
|
actorName: run.operator,
|
|
after: {
|
|
event,
|
|
channel: notification.channel,
|
|
reason: notification.reason,
|
|
},
|
|
});
|
|
}
|
|
|
|
private requireRun(
|
|
id: string,
|
|
run: DeployRunSummary | null,
|
|
): DeployRunSummary {
|
|
if (!run) {
|
|
throw new AppError(
|
|
'RESOURCE_NOT_FOUND',
|
|
`Deploy run ${id} not found`,
|
|
404,
|
|
{
|
|
id,
|
|
},
|
|
);
|
|
}
|
|
|
|
return run;
|
|
}
|
|
|
|
private toFailureSummary(error: unknown): string {
|
|
if (error instanceof Error) {
|
|
return redactSensitive({ message: error.message }).message;
|
|
}
|
|
|
|
return 'Unknown deploy execution error';
|
|
}
|
|
}
|