Skip to content

Commit ddb1143

Browse files
author
Uros Marolt
authored
Enhancement/queue prioritization c 2633 (CrowdDotDev#1900)
1 parent 61c0a6f commit ddb1143

File tree

197 files changed

+3318
-3010
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

197 files changed

+3318
-3010
lines changed

‎.github/actions/deploy-service/action.yaml

+27-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,16 @@ inputs:
1414
description: To which cloud cluster to deploy
1515
required: true
1616

17+
prioritized:
18+
description: Is the service listening on prioritized queues?
19+
required: false
20+
default: "false"
21+
22+
only_normal_queue_level:
23+
description: Is the service prioritized but only listening on normal level? (staging/lf)
24+
required: false
25+
default: "false"
26+
1727
runs:
1828
using: composite
1929
steps:
@@ -25,10 +35,25 @@ runs:
2535
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
2636
AWS_REGION: ${{ env.AWS_REGION }}
2737

28-
- name: Deploy image
38+
- name: Deploy image (non prioritized)
39+
if: inputs.prioritized == "false"
2940
shell: bash
3041
run: kubectl set image deployments/${{ inputs.service }}-dpl ${{ inputs.service }}=${{ inputs.image }}
3142

43+
- name: Deploy image (prioritized)
44+
if: inputs.prioritized == "true" && inputs.only_normal_queue_level == "false"
45+
shell: bash
46+
run: |
47+
kubectl set image deployments/${{ inputs.service }}-system-dpl ${{ inputs.service }}-system=${{ inputs.image }}
48+
kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
49+
kubectl set image deployments/${{ inputs.service }}-high-dpl ${{ inputs.service }}-high=${{ inputs.image }}
50+
kubectl set image deployments/${{ inputs.service }}-urgent-dpl ${{ inputs.service }}-urgent=${{ inputs.image }}
51+
52+
- name: Deploy image (prioritized - normal only)
53+
if: inputs.prioritized == "true" && inputs.only_normal_queue_level == "true"
54+
shell: bash
55+
run: kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
56+
3257
- uses: ./.github/actions/slack-notify
3358
with:
34-
message: 'Service *${{ inputs.service }}* was just deployed using docker image `${{ inputs.image }}`'
59+
message: "Service *${{ inputs.service }}* was just deployed using docker image `${{ inputs.image }}`"

‎.github/workflows/lf-production-deploy-new.yaml

+8-4
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ on:
88
required: true
99
type: boolean
1010
deploy_search_sync_api:
11-
description: Deploy search-sync-api service?
12-
required: true
13-
type: boolean
11+
description: Deploy search-sync-api service?
12+
required: true
13+
type: boolean
1414
deploy_integration_sync_worker:
1515
description: Deploy integration-sync-worker service?
1616
required: true
@@ -64,7 +64,7 @@ jobs:
6464
- name: Set docker image output
6565
id: image
6666
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
67-
67+
6868
build-and-push-search-sync-api:
6969
runs-on: ubuntu-latest
7070
if: ${{ inputs.deploy_search_sync_api }}
@@ -208,6 +208,8 @@ jobs:
208208
service: search-sync-worker
209209
image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
210210
cluster: ${{ env.CROWD_CLUSTER }}
211+
only_normal_queue_level: true
212+
prioritized: true
211213

212214
deploy-search-sync-api:
213215
needs: build-and-push-search-sync-api
@@ -244,6 +246,8 @@ jobs:
244246
service: integration-sync-worker
245247
image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
246248
cluster: ${{ env.CROWD_CLUSTER }}
249+
only_normal_queue_level: true
250+
prioritized: true
247251

248252
deploy-webhook-api:
249253
needs: build-and-push-webhook-api

‎.github/workflows/lf-production-deploy-original.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@ jobs:
237237
service: nodejs-worker
238238
image: ${{ needs.build-and-push-backend.outputs.image }}
239239
cluster: ${{ env.CROWD_CLUSTER }}
240+
only_normal_queue_level: true
241+
prioritized: true
240242

241243
deploy-discord-ws:
242244
needs: build-and-push-backend
@@ -291,6 +293,8 @@ jobs:
291293
service: integration-run-worker
292294
image: ${{ needs.build-and-push-integration-run-worker.outputs.image }}
293295
cluster: ${{ env.CROWD_CLUSTER }}
296+
only_normal_queue_level: true
297+
prioritized: true
294298

295299
deploy-integration-stream-worker:
296300
needs: build-and-push-integration-stream-worker
@@ -309,6 +313,8 @@ jobs:
309313
service: integration-stream-worker
310314
image: ${{ needs.build-and-push-integration-stream-worker.outputs.image }}
311315
cluster: ${{ env.CROWD_CLUSTER }}
316+
only_normal_queue_level: true
317+
prioritized: true
312318

313319
deploy-integration-data-worker:
314320
needs: build-and-push-integration-data-worker
@@ -327,6 +333,8 @@ jobs:
327333
service: integration-data-worker
328334
image: ${{ needs.build-and-push-integration-data-worker.outputs.image }}
329335
cluster: ${{ env.CROWD_CLUSTER }}
336+
only_normal_queue_level: true
337+
prioritized: true
330338

331339
deploy-data-sink-worker:
332340
needs: build-and-push-data-sink-worker
@@ -345,6 +353,8 @@ jobs:
345353
service: data-sink-worker
346354
image: ${{ needs.build-and-push-data-sink-worker.outputs.image }}
347355
cluster: ${{ env.CROWD_CLUSTER }}
356+
only_normal_queue_level: true
357+
prioritized: true
348358

349359
deploy-frontend:
350360
needs: build-and-push-frontend

‎.github/workflows/lf-staging-deploy-backend.yaml

+7-5
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ name: LF Staging Deploy Backend services
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'backend/**'
10-
- 'services/libs/**'
11-
- '!backend/src/serverless/microservices/python/**'
9+
- "backend/**"
10+
- "services/libs/**"
11+
- "!backend/src/serverless/microservices/python/**"
1212

1313
env:
1414
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -76,6 +76,8 @@ jobs:
7676
service: nodejs-worker
7777
image: ${{ needs.build-and-push.outputs.image }}
7878
cluster: ${{ env.CROWD_CLUSTER }}
79+
only_normal_queue_level: true
80+
prioritized: true
7981

8082
deploy-job-generator:
8183
needs: build-and-push

‎.github/workflows/lf-staging-deploy-data-sink-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: LF Staging Deploy Data Sink Worker
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/data_sink_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/data_sink_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: data-sink-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

‎.github/workflows/lf-staging-deploy-integration-data-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: LF Staging Deploy Integration Data Worker
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/integration_data_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/integration_data_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: integration-data-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

‎.github/workflows/lf-staging-deploy-integration-run-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: LF Staging Deploy Integration Run Worker
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/integration_data_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/integration_data_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: integration-run-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

‎.github/workflows/lf-staging-deploy-integration-stream-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: LF Staging Deploy Integration Stream Worker
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/integration_stream_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/integration_stream_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: integration-stream-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

‎.github/workflows/lf-staging-deploy-search-sync-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: LF Staging Deploy Search Sync Worker
33
on:
44
push:
55
branches:
6-
- 'lf-staging/**'
7-
- 'lf-staging-**'
6+
- "lf-staging/**"
7+
- "lf-staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/search_sync_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/search_sync_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: search-sync-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

‎.github/workflows/production-deploy-new.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
- name: Set docker image output
6565
id: image
6666
run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
67-
67+
6868
build-and-push-search-sync-api:
6969
runs-on: ubuntu-latest
7070
if: ${{ inputs.deploy_search_sync_api }}
@@ -208,6 +208,7 @@ jobs:
208208
service: search-sync-worker
209209
image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
210210
cluster: ${{ env.CROWD_CLUSTER }}
211+
prioritized: true
211212

212213
deploy-search-sync-api:
213214
needs: build-and-push-search-sync-api
@@ -244,6 +245,7 @@ jobs:
244245
service: integration-sync-worker
245246
image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
246247
cluster: ${{ env.CROWD_CLUSTER }}
248+
prioritized: true
247249

248250
deploy-webhook-api:
249251
needs: build-and-push-webhook-api

‎.github/workflows/production-deploy-original.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ jobs:
237237
service: nodejs-worker
238238
image: ${{ needs.build-and-push-backend.outputs.image }}
239239
cluster: ${{ env.CROWD_CLUSTER }}
240+
prioritized: true
240241

241242
deploy-discord-ws:
242243
needs: build-and-push-backend
@@ -291,6 +292,7 @@ jobs:
291292
service: integration-run-worker
292293
image: ${{ needs.build-and-push-integration-run-worker.outputs.image }}
293294
cluster: ${{ env.CROWD_CLUSTER }}
295+
prioritized: true
294296

295297
deploy-integration-stream-worker:
296298
needs: build-and-push-integration-stream-worker
@@ -309,6 +311,7 @@ jobs:
309311
service: integration-stream-worker
310312
image: ${{ needs.build-and-push-integration-stream-worker.outputs.image }}
311313
cluster: ${{ env.CROWD_CLUSTER }}
314+
prioritized: true
312315

313316
deploy-integration-data-worker:
314317
needs: build-and-push-integration-data-worker
@@ -327,6 +330,7 @@ jobs:
327330
service: integration-data-worker
328331
image: ${{ needs.build-and-push-integration-data-worker.outputs.image }}
329332
cluster: ${{ env.CROWD_CLUSTER }}
333+
prioritized: true
330334

331335
deploy-data-sink-worker:
332336
needs: build-and-push-data-sink-worker
@@ -345,6 +349,7 @@ jobs:
345349
service: data-sink-worker
346350
image: ${{ needs.build-and-push-data-sink-worker.outputs.image }}
347351
cluster: ${{ env.CROWD_CLUSTER }}
352+
prioritized: true
348353

349354
deploy-frontend:
350355
needs: build-and-push-frontend

‎.github/workflows/staging-deploy-backend.yaml

+7-5
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ name: Staging Deploy Backend services
33
on:
44
push:
55
branches:
6-
- 'staging/**'
7-
- 'staging-**'
6+
- "staging/**"
7+
- "staging-**"
88
paths:
9-
- 'backend/**'
10-
- 'services/libs/**'
11-
- '!backend/src/serverless/microservices/python/**'
9+
- "backend/**"
10+
- "services/libs/**"
11+
- "!backend/src/serverless/microservices/python/**"
1212

1313
env:
1414
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -76,6 +76,8 @@ jobs:
7676
service: nodejs-worker
7777
image: ${{ needs.build-and-push.outputs.image }}
7878
cluster: ${{ env.CROWD_CLUSTER }}
79+
prioritized: true
80+
only_normal_queue_level: true
7981

8082
deploy-job-generator:
8183
needs: build-and-push

‎.github/workflows/staging-deploy-data-sink-worker.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ name: Staging Deploy Data Sink Worker
33
on:
44
push:
55
branches:
6-
- 'staging/**'
7-
- 'staging-**'
6+
- "staging/**"
7+
- "staging-**"
88
paths:
9-
- 'services/libs/**'
10-
- 'services/apps/data_sink_worker/**'
9+
- "services/libs/**"
10+
- "services/apps/data_sink_worker/**"
1111

1212
env:
1313
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,3 +58,5 @@ jobs:
5858
service: data-sink-worker
5959
image: ${{ needs.build-and-push.outputs.image }}
6060
cluster: ${{ env.CROWD_CLUSTER }}
61+
only_normal_queue_level: true
62+
prioritized: true

0 commit comments

Comments
 (0)