Skip to content

AWS SES terminal delivery #2

AWS SES terminal delivery

AWS SES terminal delivery #2

name: AWS SES terminal delivery
on:
workflow_dispatch:
inputs:
confirm_account:
description: Exact approved general-purpose AWS test account ID
type: string
required: true
retain_stack:
description: Retain the ephemeral stack only for bounded debugging
type: boolean
required: false
default: false
permissions:
contents: read
id-token: write
concurrency:
group: hayasend-aws-terminal-delivery
cancel-in-progress: false
jobs:
terminal-delivery:
name: Send, observe, and delete
runs-on: ubuntu-latest
timeout-minutes: 50
environment: aws-integration
env:
AWS_REGION: ${{ vars.AWS_TEST_REGION }}
AWS_TEST_ACCOUNT_ID: ${{ vars.AWS_TEST_ACCOUNT_ID }}
AWS_TEST_ACCOUNT_KIND: ${{ vars.AWS_TEST_ACCOUNT_KIND }}
AWS_TEST_ROLE_ARN: ${{ vars.AWS_TEST_ROLE_ARN }}
AWS_TERMINAL_IDENTITY: ${{ vars.AWS_TERMINAL_IDENTITY }}
AWS_TERMINAL_FROM: ${{ vars.AWS_TERMINAL_FROM }}
AWS_TERMINAL_TO: ${{ vars.AWS_TERMINAL_TO }}
CONFIRM_ACCOUNT: ${{ inputs.confirm_account }}
STACK_NAME: hayasend-terminal-${{ github.run_id }}-${{ github.run_attempt }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Validate production-proof boundary
id: guard
shell: bash
run: |
set -euo pipefail
test "$GITHUB_REF" = "refs/heads/main"
test "$AWS_TEST_ACCOUNT_KIND" = "general-purpose-test"
test "$AWS_TEST_ACCOUNT_ID" = "330599756148"
test "$AWS_REGION" = "ap-northeast-1"
test "$CONFIRM_ACCOUNT" = "$AWS_TEST_ACCOUNT_ID"
test "$AWS_TERMINAL_IDENTITY" = "hayasend.com"
test -n "$AWS_TERMINAL_FROM"
test -n "$AWS_TERMINAL_TO"
expected_prefix="arn:aws:iam::${AWS_TEST_ACCOUNT_ID}:role/"
if [[ "$AWS_TEST_ROLE_ARN" != "$expected_prefix"* ]]; then
echo "::error::AWS_TEST_ROLE_ARN must name a role in the approved test account."
exit 1
fi
# shellcheck disable=SC2016
node --input-type=module -e '
const identity = process.env.AWS_TERMINAL_IDENTITY.toLowerCase();
const from = process.env.AWS_TERMINAL_FROM;
const to = process.env.AWS_TERMINAL_TO;
const fromAddress = (/<([^<>]+)>$/.exec(from)?.[1] ?? from).trim().toLowerCase();
if (!/^[^\s@]+@[^\s@]+$/.test(fromAddress) || !/^[^\s@]+@[^\s@]+$/.test(to)) {
throw new Error("Terminal-proof sender or recipient is invalid.");
}
if (!fromAddress.endsWith(`@${identity}`)) {
throw new Error("Terminal-proof sender must use the verified identity.");
}
if (fromAddress === to.toLowerCase()) {
throw new Error("Terminal-proof sender and recipient must differ.");
}
'
echo "validated=true" >> "$GITHUB_OUTPUT"
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 24.18.0
cache: npm
- uses: aws-actions/setup-sam@89ddb14d60e682855e3fea4be85b3c56485de310 # v3
with:
version: 1.164.0
- name: Install current pinned toolchain
shell: bash
run: |
set -euo pipefail
npm install --global --ignore-scripts npm@12.0.2
npm ci
{
node --version
npm --version
sam --version
aws --version
} 2>&1 | tee "$RUNNER_TEMP/aws-terminal-tool-versions.txt"
- name: Configure short-lived AWS credentials
id: aws-credentials
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
with:
role-to-assume: ${{ env.AWS_TEST_ROLE_ARN }}
aws-region: ${{ env.AWS_REGION }}
allowed-account-ids: ${{ env.AWS_TEST_ACCOUNT_ID }}
role-session-name: HayaSend-terminal-${{ github.run_id }}-${{ github.run_attempt }}
role-duration-seconds: 3600
mask-aws-account-id: true
unset-current-credentials: true
- name: Require SES production access and verified identity
shell: bash
run: |
set -euo pipefail
actual_account="$(aws sts get-caller-identity --query Account --output text)"
test "$actual_account" = "$AWS_TEST_ACCOUNT_ID"
aws sesv2 get-account \
--region "$AWS_REGION" \
> "$RUNNER_TEMP/aws-ses-account.raw.json"
aws sesv2 get-email-identity \
--email-identity "$AWS_TERMINAL_IDENTITY" \
--region "$AWS_REGION" \
> "$RUNNER_TEMP/aws-ses-identity.raw.json"
production_access="$(
jq -r '.ProductionAccessEnabled // false' \
"$RUNNER_TEMP/aws-ses-account.raw.json"
)"
sending_enabled="$(
jq -r '.SendingEnabled // false' \
"$RUNNER_TEMP/aws-ses-account.raw.json"
)"
verification_status="$(
jq -r '.VerificationStatus // "UNKNOWN"' \
"$RUNNER_TEMP/aws-ses-identity.raw.json"
)"
verified_for_sending="$(
jq -r '.VerifiedForSendingStatus // false' \
"$RUNNER_TEMP/aws-ses-identity.raw.json"
)"
test "$production_access" = "true"
test "$sending_enabled" = "true"
test "$verification_status" = "SUCCESS"
test "$verified_for_sending" = "true"
jq -n \
--arg account "$actual_account" \
--arg region "$AWS_REGION" \
--arg identity "$AWS_TERMINAL_IDENTITY" \
--argjson account_data "$(
jq '{
production_access: (.ProductionAccessEnabled // false),
sending_enabled: (.SendingEnabled // false),
enforcement_status: (.EnforcementStatus // "UNKNOWN"),
send_quota: (.SendQuota // {})
}' "$RUNNER_TEMP/aws-ses-account.raw.json"
)" \
--argjson identity_data "$(
jq '{
identity_type: (.IdentityType // "UNKNOWN"),
verification_status: (.VerificationStatus // "UNKNOWN"),
verified_for_sending: (.VerifiedForSendingStatus // false)
}' "$RUNNER_TEMP/aws-ses-identity.raw.json"
)" \
'{
object: "aws_ses_terminal_preflight",
account: $account,
region: $region,
identity: $identity,
ses: $account_data,
sender_identity: $identity_data
}' \
> "$RUNNER_TEMP/aws-terminal-preflight.json"
- name: Plan delivery-enabled ephemeral stack
shell: bash
run: |
set -euo pipefail
npm run --silent cli -- deploy aws \
--account "$AWS_TEST_ACCOUNT_ID" \
--stack "$STACK_NAME" \
--region "$AWS_REGION" \
--api-rate-limit 2 \
--api-burst-limit 2 \
--worker-reserved-concurrency 0 \
--log-retention-days 7 \
--tag Purpose=TerminalDeliveryProof \
--tag "GitHubRun=$GITHUB_RUN_ID" \
> "$RUNNER_TEMP/aws-terminal-deploy-plan.json"
node -e '
const fs = require("node:fs");
const plan = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
if (
plan.identity?.account !== process.env.AWS_TEST_ACCOUNT_ID ||
plan.region !== process.env.AWS_REGION ||
plan.parameters.WorkerReservedConcurrency !== "1" ||
plan.ses?.production_access !== true ||
plan.ses?.sending_enabled !== true
) {
throw new Error("The reviewed deployment plan is not delivery-ready.");
}
' "$RUNNER_TEMP/aws-terminal-deploy-plan.json"
- name: Deploy reviewed ephemeral stack
shell: bash
run: |
set -euo pipefail
npm run --silent cli -- deploy aws \
--account "$AWS_TEST_ACCOUNT_ID" \
--stack "$STACK_NAME" \
--region "$AWS_REGION" \
--api-rate-limit 2 \
--api-burst-limit 2 \
--worker-reserved-concurrency 0 \
--log-retention-days 7 \
--tag Purpose=TerminalDeliveryProof \
--tag "GitHubRun=$GITHUB_RUN_ID" \
--apply \
> "$RUNNER_TEMP/aws-terminal-deploy-result.jsonl"
node -e \
'require("node:fs").readFileSync(process.argv[1], "utf8").trim().split("\n").forEach(JSON.parse)' \
"$RUNNER_TEMP/aws-terminal-deploy-result.jsonl"
- name: Capture exact resource inventory
shell: bash
run: |
set -euo pipefail
stack_output() {
aws cloudformation describe-stacks \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" \
--query "Stacks[0].Outputs[?OutputKey=='$1'].OutputValue | [0]" \
--output text
}
stack_resource() {
aws cloudformation list-stack-resources \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" \
--query "StackResourceSummaries[?LogicalResourceId=='$1'].PhysicalResourceId | [0]" \
--output text
}
aws cloudformation list-stack-resources \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" \
> "$RUNNER_TEMP/aws-terminal-resource-inventory.raw.json"
api_base_url="$(stack_output ApiBaseUrl)"
http_api_id="$(stack_resource HttpApi)"
bootstrap_secret_arn="$(stack_output BootstrapSecretArn)"
payload_bucket="$(stack_output PayloadBucketName)"
data_table="$(stack_output TableName)"
schedule_group="$(stack_output EmailScheduleGroupName)"
configuration_set="$(stack_output ConfigurationSetName)"
lambda_functions="$(
jq -r \
'[.StackResourceSummaries[] | select(.ResourceType == "AWS::Lambda::Function") | .PhysicalResourceId] | join(" ")' \
"$RUNNER_TEMP/aws-terminal-resource-inventory.raw.json"
)"
sqs_queues="$(
jq -r \
'[.StackResourceSummaries[] | select(.ResourceType == "AWS::SQS::Queue") | .PhysicalResourceId] | join(" ")' \
"$RUNNER_TEMP/aws-terminal-resource-inventory.raw.json"
)"
sns_topics="$(
jq -r \
'[.StackResourceSummaries[] | select(.ResourceType == "AWS::SNS::Topic") | .PhysicalResourceId] | join(" ")' \
"$RUNNER_TEMP/aws-terminal-resource-inventory.raw.json"
)"
for value in \
"$api_base_url" \
"$http_api_id" \
"$bootstrap_secret_arn" \
"$payload_bucket" \
"$data_table" \
"$schedule_group" \
"$configuration_set" \
"$lambda_functions" \
"$sqs_queues" \
"$sns_topics"; do
if [[ -z "$value" || "$value" = "None" ]]; then
echo "::error::A required CloudFormation resource is missing."
exit 1
fi
done
{
echo "API_BASE_URL=$api_base_url"
echo "HTTP_API_ID=$http_api_id"
echo "BOOTSTRAP_SECRET_ARN=$bootstrap_secret_arn"
echo "PAYLOAD_BUCKET=$payload_bucket"
echo "DATA_TABLE=$data_table"
echo "SCHEDULE_GROUP=$schedule_group"
echo "CONFIGURATION_SET=$configuration_set"
echo "LAMBDA_FUNCTIONS=$lambda_functions"
echo "SQS_QUEUES=$sqs_queues"
echo "SNS_TOPICS=$sns_topics"
} >> "$GITHUB_ENV"
jq '{
object: "aws_terminal_resource_inventory",
resource_counts: (
.StackResourceSummaries
| group_by(.ResourceType)
| map({key: .[0].ResourceType, value: length})
| from_entries
)
}' "$RUNNER_TEMP/aws-terminal-resource-inventory.raw.json" \
> "$RUNNER_TEMP/aws-terminal-resource-inventory.json"
- name: Send and require terminal delivery
shell: bash
run: |
set -euo pipefail
bootstrap_key="$(
aws secretsmanager get-secret-value \
--secret-id "$BOOTSTRAP_SECRET_ARN" \
--region "$AWS_REGION" \
--query SecretString \
--output text
)"
echo "::add-mask::$bootstrap_key"
HAYASEND_BASE_URL="$API_BASE_URL" \
HAYASEND_EXPECTED_API_ID="$HTTP_API_ID" \
HAYASEND_BOOTSTRAP_KEY="$bootstrap_key" \
node scripts/aws-terminal-delivery.mjs \
| tee "$RUNNER_TEMP/aws-terminal-observations.jsonl"
tail -n 1 "$RUNNER_TEMP/aws-terminal-observations.jsonl" \
> "$RUNNER_TEMP/aws-terminal-delivery.json"
email_id="$(
node -e '
const fs = require("node:fs");
const proof = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
if (
proof.object !== "aws_ses_terminal_delivery_proof" ||
proof.terminal !== true ||
proof.scoped_api_key_revoked !== true
) {
throw new Error("Terminal delivery proof is incomplete.");
}
process.stdout.write(proof.email_id);
' "$RUNNER_TEMP/aws-terminal-delivery.json"
)"
echo "HAYASEND_EMAIL_ID=$email_id" >> "$GITHUB_ENV"
- name: Correlate SES event and immutable recipient ledger
shell: bash
run: |
set -euo pipefail
HAYASEND_TABLE_NAME="$DATA_TABLE" \
node scripts/aws-terminal-ledger.mjs \
> "$RUNNER_TEMP/aws-terminal-ledger.json"
node -e '
const fs = require("node:fs");
const proof = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
if (
proof.terminal !== true ||
proof.provider_id_correlated !== true ||
proof.exact_recipient_correlated !== true
) {
throw new Error("SES ledger correlation proof is incomplete.");
}
' "$RUNNER_TEMP/aws-terminal-ledger.json"
- name: Show CloudFormation failure events
if: ${{ failure() && steps.aws-credentials.outcome == 'success' }}
shell: bash
run: |
aws cloudformation describe-stack-events \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" \
--query "StackEvents[?ResourceStatusReason!=null].[Timestamp,LogicalResourceId,ResourceStatus,ResourceStatusReason] | [:20]" \
--output table || true
- name: Delete ephemeral resources
id: cleanup
if: ${{ always() && !inputs.retain_stack && steps.aws-credentials.outcome == 'success' }}
shell: bash
run: |
set -euo pipefail
stack_exists=false
if aws cloudformation describe-stacks \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" >/dev/null 2>&1; then
stack_exists=true
fi
resource_id() {
aws cloudformation list-stack-resources \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION" \
--query "StackResourceSummaries[?LogicalResourceId=='$1'].PhysicalResourceId | [0]" \
--output text
}
if [[ "$stack_exists" = true ]]; then
if [[ -z "${PAYLOAD_BUCKET:-}" || "$PAYLOAD_BUCKET" = "None" ]]; then
PAYLOAD_BUCKET="$(resource_id PayloadBucket)"
fi
if [[ -z "${DATA_TABLE:-}" || "$DATA_TABLE" = "None" ]]; then
DATA_TABLE="$(resource_id DataTable)"
fi
if [[ -z "${SCHEDULE_GROUP:-}" || "$SCHEDULE_GROUP" = "None" ]]; then
SCHEDULE_GROUP="$(resource_id EmailScheduleGroup)"
fi
if [[ -z "${CONFIGURATION_SET:-}" || "$CONFIGURATION_SET" = "None" ]]; then
CONFIGURATION_SET="$(resource_id ConfigurationSet)"
fi
if [[ -n "${SCHEDULE_GROUP:-}" && "$SCHEDULE_GROUP" != "None" ]]; then
schedule_names="$(
aws scheduler list-schedules \
--group-name "$SCHEDULE_GROUP" \
--region "$AWS_REGION" \
--query "Schedules[].Name" \
--output text 2>/dev/null || true
)"
for schedule_name in $schedule_names; do
aws scheduler delete-schedule \
--group-name "$SCHEDULE_GROUP" \
--name "$schedule_name" \
--region "$AWS_REGION"
done
fi
aws cloudformation delete-stack \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION"
aws cloudformation wait stack-delete-complete \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION"
fi
if [[ -n "${PAYLOAD_BUCKET:-}" && "$PAYLOAD_BUCKET" != "None" ]] &&
aws s3api head-bucket --bucket "$PAYLOAD_BUCKET" >/dev/null 2>&1; then
aws s3 rm "s3://$PAYLOAD_BUCKET" --recursive
aws s3api delete-bucket \
--bucket "$PAYLOAD_BUCKET" \
--region "$AWS_REGION"
fi
if [[ -n "${DATA_TABLE:-}" && "$DATA_TABLE" != "None" ]] &&
aws dynamodb describe-table \
--table-name "$DATA_TABLE" \
--region "$AWS_REGION" >/dev/null 2>&1; then
aws dynamodb delete-table \
--table-name "$DATA_TABLE" \
--region "$AWS_REGION" >/dev/null
aws dynamodb wait table-not-exists \
--table-name "$DATA_TABLE" \
--region "$AWS_REGION"
fi
legacy_log_groups="$(
aws logs describe-log-groups \
--log-group-name-prefix "/aws/lambda/${STACK_NAME}-" \
--region "$AWS_REGION" \
--query "logGroups[].logGroupName" \
--output text 2>/dev/null || true
)"
for log_group in $legacy_log_groups; do
aws logs delete-log-group \
--log-group-name "$log_group" \
--region "$AWS_REGION"
done
- name: Verify zero run-scoped residue
if: ${{ always() && !inputs.retain_stack && steps.aws-credentials.outcome == 'success' }}
shell: bash
run: |
set -euo pipefail
residue=0
absent() {
local label="$1"
shift
if "$@" >/dev/null 2>&1; then
echo "::error::$label remains after cleanup."
residue=$((residue + 1))
fi
}
absent "CloudFormation stack" \
aws cloudformation describe-stacks \
--stack-name "$STACK_NAME" \
--region "$AWS_REGION"
if [[ -n "${PAYLOAD_BUCKET:-}" && "$PAYLOAD_BUCKET" != "None" ]]; then
absent "S3 payload bucket" \
aws s3api head-bucket --bucket "$PAYLOAD_BUCKET"
fi
if [[ -n "${DATA_TABLE:-}" && "$DATA_TABLE" != "None" ]]; then
absent "DynamoDB table" \
aws dynamodb describe-table \
--table-name "$DATA_TABLE" \
--region "$AWS_REGION"
fi
if [[ -n "${HTTP_API_ID:-}" && "$HTTP_API_ID" != "None" ]]; then
absent "API Gateway API" \
aws apigatewayv2 get-api \
--api-id "$HTTP_API_ID" \
--region "$AWS_REGION"
fi
for function_name in ${LAMBDA_FUNCTIONS:-}; do
absent "Lambda function $function_name" \
aws lambda get-function \
--function-name "$function_name" \
--region "$AWS_REGION"
done
for queue_url in ${SQS_QUEUES:-}; do
absent "SQS queue" \
aws sqs get-queue-attributes \
--queue-url "$queue_url" \
--attribute-names QueueArn \
--region "$AWS_REGION"
done
for topic_arn in ${SNS_TOPICS:-}; do
absent "SNS topic" \
aws sns get-topic-attributes \
--topic-arn "$topic_arn" \
--region "$AWS_REGION"
done
if [[ -n "${SCHEDULE_GROUP:-}" && "$SCHEDULE_GROUP" != "None" ]]; then
absent "Scheduler group" \
aws scheduler get-schedule-group \
--name "$SCHEDULE_GROUP" \
--region "$AWS_REGION"
fi
if [[ -n "${BOOTSTRAP_SECRET_ARN:-}" && "$BOOTSTRAP_SECRET_ARN" != "None" ]]; then
absent "Secrets Manager bootstrap secret" \
aws secretsmanager describe-secret \
--secret-id "$BOOTSTRAP_SECRET_ARN" \
--region "$AWS_REGION"
fi
if [[ -n "${CONFIGURATION_SET:-}" && "$CONFIGURATION_SET" != "None" ]]; then
absent "SES configuration set" \
aws sesv2 get-configuration-set \
--configuration-set-name "$CONFIGURATION_SET" \
--region "$AWS_REGION"
fi
lambda_residue="$(
aws lambda list-functions \
--region "$AWS_REGION" \
--query "Functions[?starts_with(FunctionName, '$STACK_NAME-')].FunctionName" \
--output text
)"
managed_log_residue="$(
aws logs describe-log-groups \
--log-group-name-prefix "/hayasend/$STACK_NAME/" \
--region "$AWS_REGION" \
--query "logGroups[].logGroupName" \
--output text
)"
legacy_log_residue="$(
aws logs describe-log-groups \
--log-group-name-prefix "/aws/lambda/${STACK_NAME}-" \
--region "$AWS_REGION" \
--query "logGroups[].logGroupName" \
--output text
)"
if [[ -n "$lambda_residue" || -n "$managed_log_residue" || -n "$legacy_log_residue" ]]; then
echo "::error::Run-scoped Lambda or log-group residue remains."
residue=$((residue + 1))
fi
jq -n \
--arg stack "$STACK_NAME" \
--arg region "$AWS_REGION" \
--argjson residue "$residue" \
'{
object: "aws_terminal_cleanup_proof",
stack: $stack,
region: $region,
residue_count: $residue,
complete: ($residue == 0)
}' \
> "$RUNNER_TEMP/aws-terminal-cleanup.json"
test "$residue" -eq 0
- name: Record retained stack
if: ${{ always() && inputs.retain_stack && steps.aws-credentials.outcome == 'success' }}
shell: bash
run: |
{
echo "### AWS SES terminal-delivery resources retained"
echo
echo "Stack: \`$STACK_NAME\`"
echo "Region: \`$AWS_REGION\`"
echo
echo "Delete it manually as soon as bounded debugging is complete."
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload metadata-only terminal evidence
if: ${{ always() && steps.guard.outputs.validated == 'true' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: aws-terminal-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/aws-terminal-*.json
${{ runner.temp }}/aws-terminal-*.jsonl
${{ runner.temp }}/aws-terminal-*.txt
if-no-files-found: error
retention-days: 30