feat: 接入通知 outbox BullMQ 调度
- integrations/redis: 新增 BullMQ 连接参数工厂并复用 Redis 模块导出\n- notifications/wecom: 优先使用 BullMQ 周期调度 outbox 重试,Redis 不可用时降级进程内定时器\n- README/RTK/.env.example: 同步通知 outbox 重试和 Redis/BullMQ 配置说明
This commit is contained in:
+6
-1
@@ -10,7 +10,7 @@ DATABASE_URL=mysql://devops_user:devops_password@127.0.0.1:3306/devops_platform
|
|||||||
USE_DATABASE_READS=false
|
USE_DATABASE_READS=false
|
||||||
PRISMA_CONNECT_ON_BOOT=false
|
PRISMA_CONNECT_ON_BOOT=false
|
||||||
|
|
||||||
# Redis and queues, reserved for async polling/notification workers.
|
# Redis and queues. Used by health checks and BullMQ-backed notification outbox retry.
|
||||||
REDIS_URL=redis://127.0.0.1:6379/0
|
REDIS_URL=redis://127.0.0.1:6379/0
|
||||||
|
|
||||||
# Jenkins integration, server-side only.
|
# Jenkins integration, server-side only.
|
||||||
@@ -34,6 +34,11 @@ NOTIFICATION_PROVIDER=replace-with-provider
|
|||||||
NOTIFICATION_WEBHOOK_URL=https://notification.example.internal/webhook/replace-with-secret-manager-value
|
NOTIFICATION_WEBHOOK_URL=https://notification.example.internal/webhook/replace-with-secret-manager-value
|
||||||
FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/replace-with-secret-manager-value
|
FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/replace-with-secret-manager-value
|
||||||
WECOM_WEBHOOK_URL=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=replace-with-secret-manager-value
|
WECOM_WEBHOOK_URL=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=replace-with-secret-manager-value
|
||||||
|
NOTIFICATION_OUTBOX_RETRY_ENABLED=false
|
||||||
|
NOTIFICATION_OUTBOX_RETRY_INTERVAL_MS=30000
|
||||||
|
NOTIFICATION_OUTBOX_RETRY_DELAY_MS=60000
|
||||||
|
NOTIFICATION_OUTBOX_MAX_ATTEMPTS=3
|
||||||
|
NOTIFICATION_OUTBOX_BATCH_SIZE=10
|
||||||
|
|
||||||
# LLM proxy configuration, server-side only.
|
# LLM proxy configuration, server-side only.
|
||||||
LLM_BASE_URL=https://llm.example.internal/v1
|
LLM_BASE_URL=https://llm.example.internal/v1
|
||||||
|
|||||||
@@ -103,10 +103,11 @@ pnpm prisma:seed
|
|||||||
- `DATABASE_URL`:MySQL 连接地址。
|
- `DATABASE_URL`:MySQL 连接地址。
|
||||||
- `USE_DATABASE_READS`:是否启用 Prisma 数据读写。
|
- `USE_DATABASE_READS`:是否启用 Prisma 数据读写。
|
||||||
- `PRISMA_CONNECT_ON_BOOT`:启动时是否主动连接数据库。
|
- `PRISMA_CONNECT_ON_BOOT`:启动时是否主动连接数据库。
|
||||||
- `REDIS_URL`:Redis 连接地址,`/health` 会执行真实 PING;后续 BullMQ worker 复用该连接边界。
|
- `REDIS_URL`:Redis 连接地址,`/health` 会执行真实 PING,通知 outbox 重试会优先使用 BullMQ 调度。
|
||||||
- `JENKINS_BASE_URL`、`JENKINS_USERNAME`、`JENKINS_API_TOKEN`:Jenkins 集成配置。
|
- `JENKINS_BASE_URL`、`JENKINS_USERNAME`、`JENKINS_API_TOKEN`:Jenkins 集成配置。
|
||||||
- `GITEA_BASE_URL`、`GITEA_TOKEN`、`GITEA_WEBHOOK_SECRET`、`GITEA_WEBHOOK_RELAY_BASE_URL`:Gitea 集成与 Jenkins relay 诊断配置。
|
- `GITEA_BASE_URL`、`GITEA_TOKEN`、`GITEA_WEBHOOK_SECRET`、`GITEA_WEBHOOK_RELAY_BASE_URL`:Gitea 集成与 Jenkins relay 诊断配置。
|
||||||
- `NOTIFICATION_PROVIDER`、`WECOM_WEBHOOK_URL`、`FEISHU_WEBHOOK_URL`、`NOTIFICATION_WEBHOOK_URL`:通知集成配置。
|
- `NOTIFICATION_PROVIDER`、`WECOM_WEBHOOK_URL`、`FEISHU_WEBHOOK_URL`、`NOTIFICATION_WEBHOOK_URL`:通知集成配置。
|
||||||
|
- `NOTIFICATION_OUTBOX_RETRY_ENABLED`、`NOTIFICATION_OUTBOX_RETRY_INTERVAL_MS`、`NOTIFICATION_OUTBOX_RETRY_DELAY_MS`、`NOTIFICATION_OUTBOX_MAX_ATTEMPTS`、`NOTIFICATION_OUTBOX_BATCH_SIZE`:通知 outbox 自动重试配置;配置 `REDIS_URL` 时使用 BullMQ 调度,否则降级为进程内定时器。
|
||||||
- `LLM_BASE_URL`、`LLM_API_KEY`、`LLM_MODEL`:Agent 配置的环境变量后备值。
|
- `LLM_BASE_URL`、`LLM_API_KEY`、`LLM_MODEL`:Agent 配置的环境变量后备值。
|
||||||
|
|
||||||
## Agent 边界
|
## Agent 边界
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
- `deploy-runs`:发布记录 API、状态模型和 BPMN 步骤摘要。
|
- `deploy-runs`:发布记录 API、状态模型和 BPMN 步骤摘要。
|
||||||
- `integrations/jenkins`:Jenkins API 客户端边界。
|
- `integrations/jenkins`:Jenkins API 客户端边界。
|
||||||
- `integrations/gitea`:Gitea API 客户端边界。
|
- `integrations/gitea`:Gitea API 客户端边界。
|
||||||
- `integrations/redis`:Redis PING 健康检查边界,后续 BullMQ worker 复用该模块。
|
- `integrations/redis`:Redis PING 健康检查和 BullMQ 连接参数边界,通知 outbox 调度复用该模块。
|
||||||
- `notifications/wecom`:企微、飞书和通用 webhook 通知边界。
|
- `notifications/wecom`:企微、飞书和通用 webhook 通知边界。
|
||||||
- `agent-config`:Agent 系统级配置、密钥加密和连接测试。
|
- `agent-config`:Agent 系统级配置、密钥加密和连接测试。
|
||||||
- `agent`:只面向运维工作流的 LLM 代理边界。
|
- `agent`:只面向运维工作流的 LLM 代理边界。
|
||||||
|
|||||||
@@ -30,6 +30,7 @@
|
|||||||
"@nestjs/platform-express": "^11.1.6",
|
"@nestjs/platform-express": "^11.1.6",
|
||||||
"@nestjs/swagger": "^11.2.0",
|
"@nestjs/swagger": "^11.2.0",
|
||||||
"@prisma/client": "^6.14.0",
|
"@prisma/client": "^6.14.0",
|
||||||
|
"bullmq": "^5.78.0",
|
||||||
"ioredis": "^5.11.1",
|
"ioredis": "^5.11.1",
|
||||||
"pino": "^10.3.1",
|
"pino": "^10.3.1",
|
||||||
"reflect-metadata": "^0.2.2",
|
"reflect-metadata": "^0.2.2",
|
||||||
|
|||||||
Generated
+164
@@ -26,6 +26,9 @@ importers:
|
|||||||
'@prisma/client':
|
'@prisma/client':
|
||||||
specifier: ^6.14.0
|
specifier: ^6.14.0
|
||||||
version: 6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3)
|
version: 6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3)
|
||||||
|
bullmq:
|
||||||
|
specifier: ^5.78.0
|
||||||
|
version: 5.78.0
|
||||||
ioredis:
|
ioredis:
|
||||||
specifier: ^5.11.1
|
specifier: ^5.11.1
|
||||||
version: 5.11.1
|
version: 5.11.1
|
||||||
@@ -513,6 +516,9 @@ packages:
|
|||||||
'@ioredis/commands@1.10.0':
|
'@ioredis/commands@1.10.0':
|
||||||
resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==}
|
resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==}
|
||||||
|
|
||||||
|
'@ioredis/commands@1.5.1':
|
||||||
|
resolution: {integrity: sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==}
|
||||||
|
|
||||||
'@isaacs/cliui@8.0.2':
|
'@isaacs/cliui@8.0.2':
|
||||||
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
||||||
engines: {node: '>=12'}
|
engines: {node: '>=12'}
|
||||||
@@ -636,6 +642,36 @@ packages:
|
|||||||
'@microsoft/tsdoc@0.16.0':
|
'@microsoft/tsdoc@0.16.0':
|
||||||
resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==}
|
resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==}
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4':
|
||||||
|
resolution: {integrity: sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==}
|
||||||
|
cpu: [arm64]
|
||||||
|
os: [darwin]
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4':
|
||||||
|
resolution: {integrity: sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [darwin]
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4':
|
||||||
|
resolution: {integrity: sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==}
|
||||||
|
cpu: [arm64]
|
||||||
|
os: [linux]
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4':
|
||||||
|
resolution: {integrity: sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==}
|
||||||
|
cpu: [arm]
|
||||||
|
os: [linux]
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4':
|
||||||
|
resolution: {integrity: sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [linux]
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4':
|
||||||
|
resolution: {integrity: sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [win32]
|
||||||
|
|
||||||
'@napi-rs/wasm-runtime@1.1.5':
|
'@napi-rs/wasm-runtime@1.1.5':
|
||||||
resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==}
|
resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==}
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
@@ -1288,6 +1324,15 @@ packages:
|
|||||||
buffer@5.7.1:
|
buffer@5.7.1:
|
||||||
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
|
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
|
||||||
|
|
||||||
|
bullmq@5.78.0:
|
||||||
|
resolution: {integrity: sha512-tT9jJmbobk9ueEfFc22egLmgwCcMGgOjZ5Y1cvgczBPv1JUmC7iHQVbQtqku2YBE5dE9uzdVpxIrBvL/YAjGwA==}
|
||||||
|
engines: {node: '>=12.22.0'}
|
||||||
|
peerDependencies:
|
||||||
|
redis: '>=5.0.0'
|
||||||
|
peerDependenciesMeta:
|
||||||
|
redis:
|
||||||
|
optional: true
|
||||||
|
|
||||||
busboy@1.6.0:
|
busboy@1.6.0:
|
||||||
resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
|
resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
|
||||||
engines: {node: '>=10.16.0'}
|
engines: {node: '>=10.16.0'}
|
||||||
@@ -1465,6 +1510,10 @@ packages:
|
|||||||
create-require@1.1.1:
|
create-require@1.1.1:
|
||||||
resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
|
resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
|
||||||
|
|
||||||
|
cron-parser@4.9.0:
|
||||||
|
resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}
|
||||||
|
engines: {node: '>=12.0.0'}
|
||||||
|
|
||||||
cross-spawn@7.0.6:
|
cross-spawn@7.0.6:
|
||||||
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
||||||
engines: {node: '>= 8'}
|
engines: {node: '>= 8'}
|
||||||
@@ -1514,6 +1563,10 @@ packages:
|
|||||||
destr@2.0.5:
|
destr@2.0.5:
|
||||||
resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==}
|
resolution: {integrity: sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==}
|
||||||
|
|
||||||
|
detect-libc@2.1.2:
|
||||||
|
resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}
|
||||||
|
engines: {node: '>=8'}
|
||||||
|
|
||||||
detect-newline@3.1.0:
|
detect-newline@3.1.0:
|
||||||
resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==}
|
resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==}
|
||||||
engines: {node: '>=8'}
|
engines: {node: '>=8'}
|
||||||
@@ -1914,6 +1967,10 @@ packages:
|
|||||||
inherits@2.0.4:
|
inherits@2.0.4:
|
||||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||||
|
|
||||||
|
ioredis@5.10.1:
|
||||||
|
resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==}
|
||||||
|
engines: {node: '>=12.22.0'}
|
||||||
|
|
||||||
ioredis@5.11.1:
|
ioredis@5.11.1:
|
||||||
resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==}
|
resolution: {integrity: sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==}
|
||||||
engines: {node: '>=12.22.0'}
|
engines: {node: '>=12.22.0'}
|
||||||
@@ -2198,6 +2255,12 @@ packages:
|
|||||||
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
|
resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
||||||
|
lodash.defaults@4.2.0:
|
||||||
|
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
|
||||||
|
|
||||||
|
lodash.isarguments@3.1.0:
|
||||||
|
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
|
||||||
|
|
||||||
lodash.memoize@4.1.2:
|
lodash.memoize@4.1.2:
|
||||||
resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==}
|
resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==}
|
||||||
|
|
||||||
@@ -2221,6 +2284,10 @@ packages:
|
|||||||
lru-cache@5.1.1:
|
lru-cache@5.1.1:
|
||||||
resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
|
resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
|
||||||
|
|
||||||
|
luxon@3.7.2:
|
||||||
|
resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==}
|
||||||
|
engines: {node: '>=12'}
|
||||||
|
|
||||||
magic-string@0.30.17:
|
magic-string@0.30.17:
|
||||||
resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==}
|
resolution: {integrity: sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==}
|
||||||
|
|
||||||
@@ -2298,6 +2365,13 @@ packages:
|
|||||||
ms@2.1.3:
|
ms@2.1.3:
|
||||||
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
|
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
|
||||||
|
|
||||||
|
msgpackr-extract@3.0.4:
|
||||||
|
resolution: {integrity: sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
|
msgpackr@2.0.2:
|
||||||
|
resolution: {integrity: sha512-c5hYOXFbP79Slh6Dzd2wzk+jnV7mX1UxfMYtilnY1NmalXPqG8DGb5cYCMBrW4AsH3zekBBZd4QrKz9NhtvYLQ==}
|
||||||
|
|
||||||
multer@2.1.1:
|
multer@2.1.1:
|
||||||
resolution: {integrity: sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==}
|
resolution: {integrity: sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==}
|
||||||
engines: {node: '>= 10.16.0'}
|
engines: {node: '>= 10.16.0'}
|
||||||
@@ -2330,6 +2404,10 @@ packages:
|
|||||||
node-fetch-native@1.6.7:
|
node-fetch-native@1.6.7:
|
||||||
resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==}
|
resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==}
|
||||||
|
|
||||||
|
node-gyp-build-optional-packages@5.2.2:
|
||||||
|
resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
node-int64@0.4.0:
|
node-int64@0.4.0:
|
||||||
resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==}
|
resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==}
|
||||||
|
|
||||||
@@ -2634,6 +2712,11 @@ packages:
|
|||||||
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
|
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
semver@7.8.0:
|
||||||
|
resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==}
|
||||||
|
engines: {node: '>=10'}
|
||||||
|
hasBin: true
|
||||||
|
|
||||||
semver@7.8.4:
|
semver@7.8.4:
|
||||||
resolution: {integrity: sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==}
|
resolution: {integrity: sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
@@ -3575,6 +3658,8 @@ snapshots:
|
|||||||
|
|
||||||
'@ioredis/commands@1.10.0': {}
|
'@ioredis/commands@1.10.0': {}
|
||||||
|
|
||||||
|
'@ioredis/commands@1.5.1': {}
|
||||||
|
|
||||||
'@isaacs/cliui@8.0.2':
|
'@isaacs/cliui@8.0.2':
|
||||||
dependencies:
|
dependencies:
|
||||||
string-width: 5.1.2
|
string-width: 5.1.2
|
||||||
@@ -3805,6 +3890,24 @@ snapshots:
|
|||||||
|
|
||||||
'@microsoft/tsdoc@0.16.0': {}
|
'@microsoft/tsdoc@0.16.0': {}
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-x64@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
'@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4':
|
||||||
|
optional: true
|
||||||
|
|
||||||
'@napi-rs/wasm-runtime@1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)':
|
'@napi-rs/wasm-runtime@1.1.5(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@emnapi/core': 1.10.0
|
'@emnapi/core': 1.10.0
|
||||||
@@ -4526,6 +4629,17 @@ snapshots:
|
|||||||
base64-js: 1.5.1
|
base64-js: 1.5.1
|
||||||
ieee754: 1.2.1
|
ieee754: 1.2.1
|
||||||
|
|
||||||
|
bullmq@5.78.0:
|
||||||
|
dependencies:
|
||||||
|
cron-parser: 4.9.0
|
||||||
|
ioredis: 5.10.1
|
||||||
|
msgpackr: 2.0.2
|
||||||
|
node-abort-controller: 3.1.1
|
||||||
|
semver: 7.8.0
|
||||||
|
tslib: 2.8.1
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
busboy@1.6.0:
|
busboy@1.6.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
streamsearch: 1.1.0
|
streamsearch: 1.1.0
|
||||||
@@ -4674,6 +4788,10 @@ snapshots:
|
|||||||
|
|
||||||
create-require@1.1.1: {}
|
create-require@1.1.1: {}
|
||||||
|
|
||||||
|
cron-parser@4.9.0:
|
||||||
|
dependencies:
|
||||||
|
luxon: 3.7.2
|
||||||
|
|
||||||
cross-spawn@7.0.6:
|
cross-spawn@7.0.6:
|
||||||
dependencies:
|
dependencies:
|
||||||
path-key: 3.1.1
|
path-key: 3.1.1
|
||||||
@@ -4704,6 +4822,9 @@ snapshots:
|
|||||||
|
|
||||||
destr@2.0.5: {}
|
destr@2.0.5: {}
|
||||||
|
|
||||||
|
detect-libc@2.1.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
detect-newline@3.1.0: {}
|
detect-newline@3.1.0: {}
|
||||||
|
|
||||||
diff@4.0.4: {}
|
diff@4.0.4: {}
|
||||||
@@ -5148,6 +5269,20 @@ snapshots:
|
|||||||
|
|
||||||
inherits@2.0.4: {}
|
inherits@2.0.4: {}
|
||||||
|
|
||||||
|
ioredis@5.10.1:
|
||||||
|
dependencies:
|
||||||
|
'@ioredis/commands': 1.5.1
|
||||||
|
cluster-key-slot: 1.1.1
|
||||||
|
debug: 4.4.3
|
||||||
|
denque: 2.1.0
|
||||||
|
lodash.defaults: 4.2.0
|
||||||
|
lodash.isarguments: 3.1.0
|
||||||
|
redis-errors: 1.2.0
|
||||||
|
redis-parser: 3.0.0
|
||||||
|
standard-as-callback: 2.1.0
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
ioredis@5.11.1:
|
ioredis@5.11.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@ioredis/commands': 1.10.0
|
'@ioredis/commands': 1.10.0
|
||||||
@@ -5605,6 +5740,10 @@ snapshots:
|
|||||||
dependencies:
|
dependencies:
|
||||||
p-locate: 5.0.0
|
p-locate: 5.0.0
|
||||||
|
|
||||||
|
lodash.defaults@4.2.0: {}
|
||||||
|
|
||||||
|
lodash.isarguments@3.1.0: {}
|
||||||
|
|
||||||
lodash.memoize@4.1.2: {}
|
lodash.memoize@4.1.2: {}
|
||||||
|
|
||||||
lodash.merge@4.6.2: {}
|
lodash.merge@4.6.2: {}
|
||||||
@@ -5624,6 +5763,8 @@ snapshots:
|
|||||||
dependencies:
|
dependencies:
|
||||||
yallist: 3.1.1
|
yallist: 3.1.1
|
||||||
|
|
||||||
|
luxon@3.7.2: {}
|
||||||
|
|
||||||
magic-string@0.30.17:
|
magic-string@0.30.17:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@jridgewell/sourcemap-codec': 1.5.5
|
'@jridgewell/sourcemap-codec': 1.5.5
|
||||||
@@ -5684,6 +5825,22 @@ snapshots:
|
|||||||
|
|
||||||
ms@2.1.3: {}
|
ms@2.1.3: {}
|
||||||
|
|
||||||
|
msgpackr-extract@3.0.4:
|
||||||
|
dependencies:
|
||||||
|
node-gyp-build-optional-packages: 5.2.2
|
||||||
|
optionalDependencies:
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.4
|
||||||
|
'@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.4
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.4
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.4
|
||||||
|
'@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.4
|
||||||
|
'@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.4
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
msgpackr@2.0.2:
|
||||||
|
optionalDependencies:
|
||||||
|
msgpackr-extract: 3.0.4
|
||||||
|
|
||||||
multer@2.1.1:
|
multer@2.1.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
append-field: 1.0.0
|
append-field: 1.0.0
|
||||||
@@ -5709,6 +5866,11 @@ snapshots:
|
|||||||
|
|
||||||
node-fetch-native@1.6.7: {}
|
node-fetch-native@1.6.7: {}
|
||||||
|
|
||||||
|
node-gyp-build-optional-packages@5.2.2:
|
||||||
|
dependencies:
|
||||||
|
detect-libc: 2.1.2
|
||||||
|
optional: true
|
||||||
|
|
||||||
node-int64@0.4.0: {}
|
node-int64@0.4.0: {}
|
||||||
|
|
||||||
node-releases@2.0.47: {}
|
node-releases@2.0.47: {}
|
||||||
@@ -5996,6 +6158,8 @@ snapshots:
|
|||||||
|
|
||||||
semver@6.3.1: {}
|
semver@6.3.1: {}
|
||||||
|
|
||||||
|
semver@7.8.0: {}
|
||||||
|
|
||||||
semver@7.8.4: {}
|
semver@7.8.4: {}
|
||||||
|
|
||||||
send@1.2.1:
|
send@1.2.1:
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
/**
|
||||||
|
* BullMQ 连接工厂统一封装 Redis URL 和 Worker 连接约束,避免业务模块散落连接参数。
|
||||||
|
*/
|
||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { ConnectionOptions } from 'bullmq';
|
||||||
|
import { EnvConfig } from '../../config/env.schema';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class BullMqConnectionFactory {
|
||||||
|
constructor(private readonly config: ConfigService<EnvConfig, true>) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建 BullMQ Queue/Worker 共用连接参数;Worker 阻塞连接要求请求重试不设上限。
|
||||||
|
*/
|
||||||
|
createConnectionOptions(
|
||||||
|
connectionName: string,
|
||||||
|
): ConnectionOptions | undefined {
|
||||||
|
const redisUrl = this.config.get('REDIS_URL', { infer: true });
|
||||||
|
|
||||||
|
if (!redisUrl) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: redisUrl,
|
||||||
|
connectionName,
|
||||||
|
connectTimeout: 2000,
|
||||||
|
maxRetriesPerRequest: null,
|
||||||
|
retryStrategy: (attempt) => Math.min(attempt * 200, 5000),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
|
import { BullMqConnectionFactory } from './bullmq-connection.factory';
|
||||||
import { RedisClient } from './redis.client';
|
import { RedisClient } from './redis.client';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
providers: [RedisClient],
|
providers: [BullMqConnectionFactory, RedisClient],
|
||||||
exports: [RedisClient],
|
exports: [BullMqConnectionFactory, RedisClient],
|
||||||
})
|
})
|
||||||
export class RedisModule {}
|
export class RedisModule {}
|
||||||
|
|||||||
@@ -1,30 +1,52 @@
|
|||||||
/**
|
/**
|
||||||
* 通知 outbox 重试器复用当前 provider 投递边界,并避免空扫描刷屏审计。
|
* 通知 outbox 重试器以 BullMQ 作为 Redis 调度层,并保留内存定时器降级路径。
|
||||||
* 后续迁移到 BullMQ 时保留“有候选才记录成功 tick”的可观测边界。
|
* 真实候选查询、投递和审计仍集中在 runOnce,避免队列层持有业务状态。
|
||||||
*/
|
*/
|
||||||
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { Queue, Worker } from 'bullmq';
|
||||||
import { AuditService } from '../../audit/audit.service';
|
import { AuditService } from '../../audit/audit.service';
|
||||||
import { redactSensitive } from '../../common/security/redact-sensitive';
|
import { redactSensitive } from '../../common/security/redact-sensitive';
|
||||||
import { EnvConfig } from '../../config/env.schema';
|
import { EnvConfig } from '../../config/env.schema';
|
||||||
|
import { BullMqConnectionFactory } from '../../integrations/redis/bullmq-connection.factory';
|
||||||
import { NotificationOutboxService } from '../outbox/notification-outbox.service';
|
import { NotificationOutboxService } from '../outbox/notification-outbox.service';
|
||||||
import { DeployNotificationService } from './deploy-notification.service';
|
import { DeployNotificationService } from './deploy-notification.service';
|
||||||
|
|
||||||
|
type NotificationOutboxRetryJobData = {
|
||||||
|
source: 'bullmq';
|
||||||
|
scheduledAt: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const retryQueueName = 'notification-outbox-retry';
|
||||||
|
const retryJobName = 'retry-tick';
|
||||||
|
const retrySchedulerId = 'notification-outbox-retry-scheduler';
|
||||||
|
const bullMqPrefix = 'devops-platform';
|
||||||
|
const bullMqStartupTimeoutMs = 5000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class NotificationOutboxRetryService
|
export class NotificationOutboxRetryService
|
||||||
implements OnModuleInit, OnModuleDestroy
|
implements OnModuleInit, OnModuleDestroy
|
||||||
{
|
{
|
||||||
private timer: ReturnType<typeof setInterval> | undefined;
|
private timer: ReturnType<typeof setInterval> | undefined;
|
||||||
|
private queue:
|
||||||
|
| Queue<NotificationOutboxRetryJobData, void, string>
|
||||||
|
| undefined;
|
||||||
|
private worker:
|
||||||
|
| Worker<NotificationOutboxRetryJobData, void, string>
|
||||||
|
| undefined;
|
||||||
private running = false;
|
private running = false;
|
||||||
|
private bullMqFallbackRecorded = false;
|
||||||
|
private bullMqRuntimeIssueRecorded = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly config: ConfigService<EnvConfig, true>,
|
private readonly config: ConfigService<EnvConfig, true>,
|
||||||
|
private readonly bullMqConnectionFactory: BullMqConnectionFactory,
|
||||||
private readonly notificationOutboxService: NotificationOutboxService,
|
private readonly notificationOutboxService: NotificationOutboxService,
|
||||||
private readonly deployNotificationService: DeployNotificationService,
|
private readonly deployNotificationService: DeployNotificationService,
|
||||||
private readonly auditService: AuditService,
|
private readonly auditService: AuditService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
onModuleInit(): void {
|
async onModuleInit(): Promise<void> {
|
||||||
if (
|
if (
|
||||||
!this.config.get('NOTIFICATION_OUTBOX_RETRY_ENABLED', { infer: true })
|
!this.config.get('NOTIFICATION_OUTBOX_RETRY_ENABLED', { infer: true })
|
||||||
) {
|
) {
|
||||||
@@ -36,19 +58,120 @@ export class NotificationOutboxRetryService
|
|||||||
{ infer: true },
|
{ infer: true },
|
||||||
);
|
);
|
||||||
|
|
||||||
|
await this.startBullMqScheduler(intervalMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
async onModuleDestroy(): Promise<void> {
|
||||||
|
this.stopMemoryScheduler();
|
||||||
|
await this.closeBullMqScheduler(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async startBullMqScheduler(intervalMs: number): Promise<void> {
|
||||||
|
const connection = this.bullMqConnectionFactory.createConnectionOptions(
|
||||||
|
'notification-outbox-retry',
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!connection) {
|
||||||
|
this.startMemoryScheduler(intervalMs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.queue = new Queue<NotificationOutboxRetryJobData, void, string>(
|
||||||
|
retryQueueName,
|
||||||
|
{
|
||||||
|
connection,
|
||||||
|
prefix: bullMqPrefix,
|
||||||
|
defaultJobOptions: {
|
||||||
|
removeOnComplete: { count: 20 },
|
||||||
|
removeOnFail: { count: 20 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
this.worker = new Worker<NotificationOutboxRetryJobData, void, string>(
|
||||||
|
retryQueueName,
|
||||||
|
async () => {
|
||||||
|
await this.runOnce();
|
||||||
|
},
|
||||||
|
{
|
||||||
|
connection,
|
||||||
|
prefix: bullMqPrefix,
|
||||||
|
concurrency: 1,
|
||||||
|
removeOnComplete: { count: 20 },
|
||||||
|
removeOnFail: { count: 20 },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
this.queue.on('error', (error) => {
|
||||||
|
void this.recordBullMqIssue('queue', error, 'bullmq_recovering');
|
||||||
|
});
|
||||||
|
this.worker.on('error', (error) => {
|
||||||
|
void this.recordBullMqIssue('worker', error, 'bullmq_recovering');
|
||||||
|
});
|
||||||
|
this.worker.on('failed', (_job, error) => {
|
||||||
|
void this.recordBullMqIssue('job', error, 'bullmq_recovering');
|
||||||
|
});
|
||||||
|
|
||||||
|
await this.withStartupTimeout(
|
||||||
|
Promise.all([
|
||||||
|
this.queue.waitUntilReady(),
|
||||||
|
this.worker.waitUntilReady(),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
await this.queue.upsertJobScheduler(
|
||||||
|
retrySchedulerId,
|
||||||
|
{ every: intervalMs },
|
||||||
|
{
|
||||||
|
name: retryJobName,
|
||||||
|
data: {
|
||||||
|
source: 'bullmq',
|
||||||
|
scheduledAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
opts: {
|
||||||
|
removeOnComplete: { count: 20 },
|
||||||
|
removeOnFail: { count: 20 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
void this.runOnce();
|
||||||
|
} catch (error) {
|
||||||
|
await this.closeBullMqScheduler(true);
|
||||||
|
await this.recordBullMqIssue('startup', error, 'memory_timer');
|
||||||
|
this.startMemoryScheduler(intervalMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private startMemoryScheduler(intervalMs: number): void {
|
||||||
|
if (this.timer) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.timer = setInterval(() => {
|
this.timer = setInterval(() => {
|
||||||
void this.runOnce();
|
void this.runOnce();
|
||||||
}, intervalMs);
|
}, intervalMs);
|
||||||
void this.runOnce();
|
void this.runOnce();
|
||||||
}
|
}
|
||||||
|
|
||||||
onModuleDestroy(): void {
|
private stopMemoryScheduler(): void {
|
||||||
if (this.timer) {
|
if (this.timer) {
|
||||||
clearInterval(this.timer);
|
clearInterval(this.timer);
|
||||||
this.timer = undefined;
|
this.timer = undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async closeBullMqScheduler(forceWorkerClose: boolean): Promise<void> {
|
||||||
|
const worker = this.worker;
|
||||||
|
const queue = this.queue;
|
||||||
|
|
||||||
|
this.worker = undefined;
|
||||||
|
this.queue = undefined;
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
worker?.close(forceWorkerClose).catch(() => undefined),
|
||||||
|
queue?.close().catch(() => undefined),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
async runOnce(): Promise<void> {
|
async runOnce(): Promise<void> {
|
||||||
if (this.running) {
|
if (this.running) {
|
||||||
return;
|
return;
|
||||||
@@ -63,13 +186,12 @@ export class NotificationOutboxRetryService
|
|||||||
const batchSize = this.config.get('NOTIFICATION_OUTBOX_BATCH_SIZE', {
|
const batchSize = this.config.get('NOTIFICATION_OUTBOX_BATCH_SIZE', {
|
||||||
infer: true,
|
infer: true,
|
||||||
});
|
});
|
||||||
const messages = await this.notificationOutboxService.listRetryableMessages(
|
const messages =
|
||||||
{
|
await this.notificationOutboxService.listRetryableMessages({
|
||||||
now: new Date(),
|
now: new Date(),
|
||||||
maxAttempts,
|
maxAttempts,
|
||||||
take: batchSize,
|
take: batchSize,
|
||||||
},
|
});
|
||||||
);
|
|
||||||
let sentCount = 0;
|
let sentCount = 0;
|
||||||
let failedCount = 0;
|
let failedCount = 0;
|
||||||
let skippedCount = 0;
|
let skippedCount = 0;
|
||||||
@@ -116,4 +238,57 @@ export class NotificationOutboxRetryService
|
|||||||
this.running = false;
|
this.running = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async withStartupTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||||
|
return Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise<T>((_resolve, reject) => {
|
||||||
|
setTimeout(
|
||||||
|
() => reject(new Error('BullMQ 调度器启动超时')),
|
||||||
|
bullMqStartupTimeoutMs,
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async recordBullMqIssue(
|
||||||
|
stage: 'startup' | 'queue' | 'worker' | 'job',
|
||||||
|
error: unknown,
|
||||||
|
mode: 'memory_timer' | 'bullmq_recovering',
|
||||||
|
): Promise<void> {
|
||||||
|
const recorded =
|
||||||
|
mode === 'memory_timer'
|
||||||
|
? this.bullMqFallbackRecorded
|
||||||
|
: this.bullMqRuntimeIssueRecorded;
|
||||||
|
|
||||||
|
if (recorded) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mode === 'memory_timer') {
|
||||||
|
this.bullMqFallbackRecorded = true;
|
||||||
|
} else {
|
||||||
|
this.bullMqRuntimeIssueRecorded = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const message =
|
||||||
|
error instanceof Error ? error.message : 'BullMQ 通知重试调度异常';
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.auditService.record({
|
||||||
|
action:
|
||||||
|
mode === 'memory_timer'
|
||||||
|
? 'NOTIFICATION_OUTBOX_RETRY_SCHEDULER_FALLBACK'
|
||||||
|
: 'NOTIFICATION_OUTBOX_RETRY_SCHEDULER_DEGRADED',
|
||||||
|
resourceType: 'notification_outbox',
|
||||||
|
after: {
|
||||||
|
stage,
|
||||||
|
mode,
|
||||||
|
message: redactSensitive({ message }).message,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// 启动早期数据库可能尚不可用,调度降级不能被审计写入失败阻断。
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
* 通知模块集中注册 provider 客户端、发布通知服务和 outbox 能力。
|
* 通知模块集中注册 provider 客户端、发布通知服务和 outbox 能力。
|
||||||
*/
|
*/
|
||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
|
import { RedisModule } from '../../integrations/redis/redis.module';
|
||||||
import { NotificationOutboxModule } from '../outbox/notification-outbox.module';
|
import { NotificationOutboxModule } from '../outbox/notification-outbox.module';
|
||||||
import { DeployNotificationService } from './deploy-notification.service';
|
import { DeployNotificationService } from './deploy-notification.service';
|
||||||
import { NotificationOutboxManualRetryController } from './notification-outbox-manual-retry.controller';
|
import { NotificationOutboxManualRetryController } from './notification-outbox-manual-retry.controller';
|
||||||
@@ -10,7 +11,7 @@ import { NotificationOutboxRetryService } from './notification-outbox-retry.serv
|
|||||||
import { WeComClient } from './wecom.client';
|
import { WeComClient } from './wecom.client';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [NotificationOutboxModule],
|
imports: [NotificationOutboxModule, RedisModule],
|
||||||
controllers: [NotificationOutboxManualRetryController],
|
controllers: [NotificationOutboxManualRetryController],
|
||||||
providers: [
|
providers: [
|
||||||
WeComClient,
|
WeComClient,
|
||||||
|
|||||||
Reference in New Issue
Block a user