Repository navigation
455 lines (419 loc) · 20.5 KB
/
Copy pathdeploy-dataops-v1.yml
File metadata and controls
455 lines (419 loc) · 20.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
name: Deploy DataOps V1 Lambda
on:
pull_request:
branches:
- main
paths:
- ".github/workflows/deploy-dataops-v1.yml"
- "backend/**"
- "mcp/**"
- "frontend/**"
- "infra/**"
- "Makefile"
- "package.json"
- "package-lock.json"
- "scripts/test-backend-transactions.sh"
- "samconfig.toml"
push:
branches:
- main
paths:
- ".github/workflows/deploy-dataops-v1.yml"
- "backend/**"
- "mcp/**"
- "frontend/**"
- "infra/**"
- "Makefile"
- "package.json"
- "package-lock.json"
- "scripts/test-backend-transactions.sh"
- "samconfig.toml"
workflow_dispatch:
permissions:
contents: read
env:
AWS_REGION: eu-west-1
AWS_DEFAULT_REGION: eu-west-1
AWS_ROLE_ARN: arn:aws:iam::817685572750:role/dataops-github-actions-deploy
DOCS_GITHUB_OWNER: DataTalksClub
DOCS_GITHUB_REPO: dataops-knowledge
DOCS_GITHUB_TOKEN_SECRET: dataops-v1.knowledge.github-token
DOCS_GITHUB_BRANCH: main
jobs:
checks:
name: Check DataOps v1 app
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Set up Node.js
uses: actions/setup-node@v6
with:
node-version-file: ".node-version"
cache: npm
cache-dependency-path: package-lock.json
- name: Install Node workspace dependencies
run: npm ci
- name: Run frontend unit coverage
run: npm run test:frontend:coverage
- name: Run backend tests
run: npm --prefix backend test
- name: Run DataOps MCP tests
run: npm run test:mcp
# The transaction suites start amazon/dynamodb-local. Pull it up front
# with retries so a transient Docker Hub/network blip on the runner does
# not turn into a red run (and a failure mail) for unrelated changes.
- name: Pull DynamoDB Local image
run: |
for attempt in 1 2 3 4 5; do
docker pull amazon/dynamodb-local:latest && exit 0
echo "docker pull failed (attempt $attempt); retrying in $((attempt * 10))s" >&2
sleep $((attempt * 10))
done
exit 1
- name: Gate all backend DynamoDB transactions
run: npm run test:backend:transactions
- name: Typecheck backend
run: npm --prefix backend run typecheck
- name: Build backend
run: npm --prefix backend run build
- name: Set up SAM
uses: aws-actions/setup-sam@v3
with:
use-installer: true
token: ${{ secrets.GITHUB_TOKEN }}
- name: Validate SAM template
run: make sam-validate
deploy:
name: Deploy DataOps v1 app
if: github.event_name != 'pull_request'
concurrency:
group: dataops-v1-deploy
cancel-in-progress: false
runs-on: ubuntu-24.04
needs: checks
permissions:
contents: read
id-token: write
env:
AUTH_BASE_URL: https://auth.dtcdev.click
AUTH_USER_POOL_ID: us-east-1_H7nJu52Bs
# AUTH_ISSUER, AUTH_JWKS_URL, and AUTH_CLIENT_ID are resolved from the
# shared-auth stack's own outputs in the "Resolve shared auth client"
# step below, not hardcoded here — a hardcoded client_id goes stale
# silently whenever the Cognito app client is recreated.
AUTH_CALLBACK_URL: https://ops.dtcdev.click/auth/callback
AUTH_LOGOUT_URL: https://ops.dtcdev.click/
# 30 days in seconds
AUTH_SESSION_LIFETIME_SECONDS: 2592000
OAUTH_ISSUER_URL: https://oauth.dtcdev.click
EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET: ${{ vars.EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET }}
EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX: ${{ vars.EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX }}
EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN: ${{ secrets.EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN }}
SPONSOR_FINANCE_ENABLED: ${{ vars.SPONSOR_FINANCE_ENABLED }}
SPONSOR_COMMUNICATION_SEND_ENABLED: ${{ vars.SPONSOR_COMMUNICATION_SEND_ENABLED }}
SPONSOR_COMMUNICATION_TEMPLATE_SECRET_ARN: ${{ vars.SPONSOR_COMMUNICATION_TEMPLATE_SECRET_ARN }}
SPONSOR_COMMUNICATION_HMAC_SECRET_ARN: ${{ vars.SPONSOR_COMMUNICATION_HMAC_SECRET_ARN }}
steps:
- name: Check out repository
uses: actions/checkout@v6
- name: Set up Node.js
uses: actions/setup-node@v6
with:
node-version-file: ".node-version"
cache: npm
cache-dependency-path: package-lock.json
- name: Set up SAM
uses: aws-actions/setup-sam@v3
with:
use-installer: true
token: ${{ secrets.GITHUB_TOKEN }}
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v6.1.0
with:
role-to-assume: ${{ env.AWS_ROLE_ARN }}
role-session-name: dataops-v1-deploy
aws-region: ${{ env.AWS_REGION }}
- name: Resolve shared auth client
run: |
auth_output() {
aws cloudformation describe-stacks --region us-east-1 --stack-name dtcdev-shared-auth \
--query "Stacks[0].Outputs[?OutputKey=='$1'].OutputValue" --output text
}
{
echo "AUTH_CLIENT_ID=$(auth_output DataOpsClientId)"
echo "AUTH_ISSUER=$(auth_output IssuerUrl)"
echo "AUTH_JWKS_URL=$(auth_output JwksUrl)"
} >> "$GITHUB_ENV"
- name: Resolve stable scheduled Lambda targets
run: |
set -euo pipefail
stack_name="dataops-v1"
caller_arn="$(aws sts get-caller-identity --query Arn --output text)"
account_id="$(aws sts get-caller-identity --query Account --output text)"
partition="${caller_arn#arn:}"
partition="${partition%%:*}"
stack_json="$(aws cloudformation describe-stacks \
--stack-name "$stack_name" \
--region "$AWS_REGION" \
--query 'Stacks[0].{StackId:StackId,Outputs:Outputs}' \
--output json)"
stack_id="$(jq -er '.StackId' <<<"$stack_json")"
case "$stack_id" in
"arn:$partition:cloudformation:$AWS_REGION:$account_id:stack/$stack_name/"*) ;;
*) echo "Existing stack identity does not match the configured stack/account/region" >&2; exit 1 ;;
esac
stack_output() {
jq -er --arg key "$1" '
[.Outputs[] | select(.OutputKey == $key) | .OutputValue]
| if length == 1 and (.[0] | type == "string" and test("^[A-Za-z0-9-_]{1,64}$"))
then .[0]
else error("required Lambda-name output is missing, ambiguous, or invalid: " + $key)
end
' <<<"$stack_json"
}
{
echo "BACKEND_TARGET_FUNCTION_NAME=$(stack_output BackendFunctionName)"
} >> "$GITHUB_ENV"
- name: Build SAM artifact
run: make sam-build
- name: Verify packaged canonical frontend
run: make verify-sam-frontend
- name: Verify packaged runtime infrastructure boundary
run: make verify-sam-runtime-boundary
- name: Test packaged frontend isolation
run: make test-sam-frontend-isolation
- name: Build and upload banner template bundle
run: |
set -euo pipefail
stack_params="$(aws cloudformation describe-stacks \
--stack-name dataops-v1 \
--region "$AWS_REGION" \
--query 'Stacks[0].Parameters' \
--output json)"
knowledge_bucket="$(jq -er '.[] | select(.ParameterKey == "KnowledgeBucketName") | .ParameterValue' <<<"$stack_params")"
current_key="$(jq -r '.[] | select(.ParameterKey == "BannerTemplateBundleKey") | .ParameterValue // empty' <<<"$stack_params")"
default_key="$(awk '/BannerTemplateBundleKey:/{found=1} found && /Default:/{print $2; exit}' infra/template.full.yaml)"
: "${current_key:=$default_key}"
aws s3 cp "s3://$knowledge_bucket/$current_key" "$RUNNER_TEMP/current-bundle.json"
if ! node -e "const b = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); process.exit(b.assets?.assetLogo ? 0 : 1)" "$RUNNER_TEMP/current-bundle.json"; then
aws s3 cp "s3://$knowledge_bucket/$default_key" "$RUNNER_TEMP/current-bundle.json"
fi
npm --prefix "$RUNNER_TEMP/tsx" install --no-save tsx >/dev/null
bundle_key="$(node --import "$RUNNER_TEMP/tsx/node_modules/tsx/dist/loader.mjs" backend/scripts/build-banner-bundle.ts \
--current "$RUNNER_TEMP/current-bundle.json" \
--out-dir "$RUNNER_TEMP/banner-bundle")"
if ! aws s3 ls "s3://$knowledge_bucket/$bundle_key" >/dev/null 2>&1; then
aws s3 cp "$RUNNER_TEMP/banner-bundle/$bundle_key" "s3://$knowledge_bucket/$bundle_key" --content-type application/json
fi
echo "BANNER_TEMPLATE_BUNDLE_KEY=$bundle_key" >> "$GITHUB_ENV"
- name: Deploy DataOps v1 stack
run: |
set -euo pipefail
: "${EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX:=transfer/}"
: "${SPONSOR_FINANCE_ENABLED:=false}"
: "${SPONSOR_COMMUNICATION_SEND_ENABLED:=false}"
if [ "$SPONSOR_COMMUNICATION_SEND_ENABLED" = "true" ] && { [ -z "$SPONSOR_COMMUNICATION_TEMPLATE_SECRET_ARN" ] || [ -z "$SPONSOR_COMMUNICATION_HMAC_SECRET_ARN" ]; }; then
echo "Sponsor reviewed sending cannot be enabled without separate template and HMAC secret ARNs" >&2
exit 1
fi
sam deploy \
--config-env full-sandbox \
--parameter-overrides \
"ParameterKey=GitHubOwner,ParameterValue=$DOCS_GITHUB_OWNER" \
"ParameterKey=GitHubRepo,ParameterValue=$DOCS_GITHUB_REPO" \
"ParameterKey=GitHubTokenSecretName,ParameterValue=$DOCS_GITHUB_TOKEN_SECRET" \
"ParameterKey=GitHubBranch,ParameterValue=$DOCS_GITHUB_BRANCH" \
"ParameterKey=AuthBaseUrl,ParameterValue=$AUTH_BASE_URL" \
"ParameterKey=AuthUserPoolId,ParameterValue=$AUTH_USER_POOL_ID" \
"ParameterKey=AuthIssuer,ParameterValue=$AUTH_ISSUER" \
"ParameterKey=AuthJwksUrl,ParameterValue=$AUTH_JWKS_URL" \
"ParameterKey=AuthClientId,ParameterValue=$AUTH_CLIENT_ID" \
"ParameterKey=AuthCallbackUrl,ParameterValue=$AUTH_CALLBACK_URL" \
"ParameterKey=AuthLogoutUrl,ParameterValue=$AUTH_LOGOUT_URL" \
"ParameterKey=AuthSessionLifetimeSeconds,ParameterValue=$AUTH_SESSION_LIFETIME_SECONDS" \
"ParameterKey=OauthIssuerUrl,ParameterValue=$OAUTH_ISSUER_URL" \
"ParameterKey=EmailDocumentExternalSourceBucketName,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET" \
"ParameterKey=EmailDocumentExternalSourcePrefix,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX" \
"ParameterKey=EmailDocumentExternalSourceKmsKeyArn,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN" \
"ParameterKey=SponsorFinanceEnabled,ParameterValue=$SPONSOR_FINANCE_ENABLED" \
"ParameterKey=DataOpsEnvironment,ParameterValue=sandbox" \
"ParameterKey=BackendTargetFunctionName,ParameterValue=$BACKEND_TARGET_FUNCTION_NAME" \
"ParameterKey=SponsorCommunicationSendEnabled,ParameterValue=$SPONSOR_COMMUNICATION_SEND_ENABLED" \
"ParameterKey=SponsorCommunicationTemplateSecretArn,ParameterValue=$SPONSOR_COMMUNICATION_TEMPLATE_SECRET_ARN" \
"ParameterKey=SponsorCommunicationHmacSecretArn,ParameterValue=$SPONSOR_COMMUNICATION_HMAC_SECRET_ARN" \
"ParameterKey=BannerTemplateBundleKey,ParameterValue=$BANNER_TEMPLATE_BUNDLE_KEY" \
- name: Project authored workflow templates
timeout-minutes: 10
env:
# BackendFunction has no ReservedConcurrentExecutions in template.full.yaml
# (unreserved steady-state concurrency, shared by BackendFunctionUrl HTTP/MCP
# traffic, the four daily EventBridge cron rules, and post-deploy invokes).
# Standard retries back off on Invoke throttling (TooManyRequestsException /
# ReservedFunctionConcurrentInvocationLimitExceeded), including a temporary
# ReservedConcurrentExecutions=0 writer-drain pause during maintenance.
# Bound attempts; FunctionError and the projection contract below still fail.
# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-retries.html
AWS_RETRY_MODE: standard
AWS_MAX_ATTEMPTS: '20'
run: |
set -euo pipefail
backend_function="$(aws cloudformation describe-stack-resource \
--stack-name dataops-v1 \
--logical-resource-id BackendFunction \
--query "StackResourceDetail.PhysicalResourceId" \
--output text)"
seed_response="$RUNNER_TEMP/dataops-runtime-seeds.json"
seed_error="$(aws lambda invoke \
--function-name "$backend_function" \
--cli-binary-format raw-in-base64-out \
--payload '{"source":"dataops.deploy","detail-type":"Runtime Seed","detail":{"dataopsAction":"sync-runtime-seeds"}}' \
--query FunctionError \
--output text \
"$seed_response")"
if [ "$seed_error" != "None" ]; then
echo "Runtime seed failed inside the deployed Lambda" >&2
exit 1
fi
if ! jq -e '
def nonnegative_integer: type == "number" and floor == . and . >= 0;
.statusCode == 200
and ((.body | fromjson) as $body
| ($body | keys == ["templates"])
and ($body.templates | keys == ["created", "deleted", "total", "unchanged", "updated"])
# Authored templates come from the private knowledge branch, whose
# valid template count changes independently of application code.
and ($body.templates.total | nonnegative_integer and . > 0)
and ([$body.templates.created, $body.templates.updated, $body.templates.unchanged, $body.templates.deleted] | all(.[]; nonnegative_integer))
and ($body.templates.created + $body.templates.updated + $body.templates.unchanged == $body.templates.total)
)
' "$seed_response" >/dev/null; then
echo "Runtime seed did not confirm the authored-template reconciliation contract" >&2
# Emit only numeric counts, never raw runtime or private knowledge data.
jq -c '
def safe_count: if type == "number" then . else null end;
try ((.body | fromjson) as $body | {
status: (.statusCode | safe_count),
templates: ($body.templates.total | safe_count)
}) catch {error: "malformed-seed-response"}
' "$seed_response" >&2 || true
exit 1
fi
jq -c '(.body | fromjson) | {templates: {created: .templates.created, deleted: .templates.deleted, total: .templates.total, unchanged: .templates.unchanged, updated: .templates.updated}}' "$seed_response"
echo "Authored workflow templates projected."
- name: Verify scheduled calendar hold expiry
timeout-minutes: 5
env:
AWS_RETRY_MODE: standard
AWS_MAX_ATTEMPTS: '20'
run: |
set -euo pipefail
backend_function="$(aws cloudformation describe-stack-resource \
--stack-name dataops-v1 \
--logical-resource-id BackendFunction \
--query "StackResourceDetail.PhysicalResourceId" \
--output text)"
dry_response="$RUNNER_TEMP/dataops-calendar-holds-expire-dry.json"
dry_error="$(aws lambda invoke \
--function-name "$backend_function" \
--cli-binary-format raw-in-base64-out \
--payload '{"source":"aws.events","detail-type":"Scheduled Event","detail":{"dataopsAction":"calendar-holds-expire","dryRun":true}}' \
--query FunctionError \
--output text \
"$dry_response")"
if [ "$dry_error" != "None" ]; then
echo "Scheduled calendar-holds-expire dryRun failed inside the deployed Lambda" >&2
exit 1
fi
jq -e '
.statusCode == 200
and ((.body | fromjson) as $body
| $body.dryRun == true
and ($body.activeCount | type == "number" and . >= 0)
and ($body.expired | type == "array")
and ($body.followUps | type == "array")
and ($body.note | type == "string")
)
' "$dry_response" >/dev/null
jq -c '(.body | fromjson) | {
dryRun: .dryRun,
activeCount: .activeCount,
expiredCount: (.expired | length),
expiredHoldIds: [.expired[].id],
followUps: [.followUps[] | {holdId: .holdId, expiredAt: .expiredAt}],
note: .note
}' "$dry_response"
live_response="$RUNNER_TEMP/dataops-calendar-holds-expire-live.json"
live_error="$(aws lambda invoke \
--function-name "$backend_function" \
--cli-binary-format raw-in-base64-out \
--payload '{"source":"aws.events","detail-type":"Scheduled Event","detail":{"dataopsAction":"calendar-holds-expire"}}' \
--query FunctionError \
--output text \
"$live_response")"
if [ "$live_error" != "None" ]; then
echo "Scheduled calendar-holds-expire live run failed inside the deployed Lambda" >&2
exit 1
fi
jq -e '
.statusCode == 200
and ((.body | fromjson) as $body
| $body.dryRun == false
and ($body.activeCount | type == "number" and . >= 0)
and ($body.expired | type == "array")
and ($body.followUps | type == "array")
and ($body.note | type == "string")
)
' "$live_response" >/dev/null
jq -c '(.body | fromjson) | {
dryRun: .dryRun,
activeCount: .activeCount,
expiredCount: (.expired | length),
expiredHoldIds: [.expired[].id],
followUps: [.followUps[] | {holdId: .holdId, taskId: .taskId, notificationId: .notificationId, expiredAt: .expiredAt}],
note: .note
}' "$live_response"
echo "Scheduled calendar hold expiry verified."
- name: Smoke test deployed single-origin backend
run: |
backend_url="$(aws cloudformation describe-stacks \
--stack-name dataops-v1 \
--query "Stacks[0].Outputs[?OutputKey=='BackendUrl'].OutputValue" \
--output text)"
# The Function URL output ends with a trailing slash; strip it so we hit
# "/" and "/login" instead of malformed double-slash paths ("//", "//login").
backend_url="${backend_url%/}"
# /login redirects to the shared Cognito authorize endpoint and root redirects locally.
# The Function URL event normalization in handler.ts must work for this.
# Retry a few times: immediately after a stack update the Function URL can
# briefly return a cold-start/propagation blip before the new code settles.
code=""
login_code=""
for attempt in 1 2 3 4 5; do
code=$(curl -sS -o /dev/null -w "%{http_code}" "$backend_url/")
login_code=$(curl -sS -o /dev/null -w "%{http_code}" "$backend_url/login")
if [ "$code" = "302" ] && [ "$login_code" = "302" ]; then
echo "Backend smoke OK on attempt $attempt: root=$code login=$login_code"
exit 0
fi
echo "Smoke attempt $attempt: root=$code login=$login_code (retrying)"
sleep 10
done
echo "Backend smoke test failed after retries: root expected 302 got $code; /login expected 302 got $login_code"
exit 1
- name: Smoke test dedicated DataOps MCP
run: |
set -euo pipefail
mcp_url="$(aws cloudformation describe-stacks \
--stack-name dataops-v1 \
--query "Stacks[0].Outputs[?OutputKey=='McpUrl'].OutputValue" \
--output text)"
mcp_origin="${mcp_url%/mcp}"
curl --fail --silent --show-error "$mcp_origin/health" \
| jq -e '.status == "ok" and .service == "dataops-mcp"' >/dev/null
code="$(curl --silent --show-error -o /dev/null -w '%{http_code}' \
-H 'Content-Type: application/json' \
--data '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' "$mcp_url")"
test "$code" = 401
echo "DataOps MCP endpoint: $mcp_url"