refactor: update KMS key handling in midaz configuration #73

Workflow file for this run

.github/workflows/ci.yml at 3720412

	name: CI - Validate CloudFormation Templates

	on:
	push:
	branches:
	- main
	- develop
	paths:
	- 'templates/**'
	- 'products/**'
	- 'infrastructure/**'
	- '.github/workflows/**'
	pull_request:
	branches:
	- main
	- develop

	permissions:
	contents: read
	pull-requests: write
	issues: write
	security-events: write
	id-token: write
	actions: read

	jobs:
	# ============================================================================
	# Shared Workflow: PR Validation
	# ============================================================================
	pr-validation:
	name: PR Validation
	if: github.event_name == 'pull_request'
	uses: LerianStudio/github-actions-shared-workflows/.github/workflows/pr-validation.yml@v1.10.2
	with:
	runner_type: blacksmith-4vcpu-ubuntu-2404
	pr_title_types: \|
	feat
	fix
	docs
	style
	refactor
	perf
	test
	chore
	ci
	build
	revert
	pr_title_scopes: \|
	vpc
	eks
	rds
	documentdb
	elasticache
	amazonmq
	route53
	alb-controller
	external-dns
	midaz
	midaz-helm
	midaz-infrastructure
	midaz-application
	midaz-complete
	scripts
	ci
	docs
	infra
	require_scope: false
	min_description_length: 30
	check_changelog: true
	enable_auto_labeler: true
	labeler_config_path: '.github/labeler.yml'
	enforce_source_branches: true
	allowed_source_branches: 'develop\|release/\|hotfix/\|feature/\|fix/'
	target_branches_for_source_check: 'main'
	secrets: inherit

	# ============================================================================
	# CloudFormation-Specific: Template Linting
	# ============================================================================
	cfn-lint:
	name: CloudFormation Lint
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install cfn-lint
	run: pip install cfn-lint

	- name: Run cfn-lint on core templates
	run: \|
	echo "## CloudFormation Lint Results" >> $GITHUB_STEP_SUMMARY
	echo "" >> $GITHUB_STEP_SUMMARY

	# Lint core templates
	echo "### Core Templates" >> $GITHUB_STEP_SUMMARY
	if cfn-lint templates/*.yaml --regions us-east-1 --ignore-checks W --format parseable; then
	echo "All core templates passed linting (errors only)" >> $GITHUB_STEP_SUMMARY
	else
	echo "Some core templates have linting errors" >> $GITHUB_STEP_SUMMARY
	exit 1
	fi

	- name: Run cfn-lint on product templates
	run: \|
	echo "### Product Templates" >> $GITHUB_STEP_SUMMARY
	for product_dir in products/*/; do
	product=$(basename "$product_dir")
	echo "#### $product" >> $GITHUB_STEP_SUMMARY
	if ls "$product_dir"*.yaml 1> /dev/null 2>&1; then
	if cfn-lint "$product_dir"*.yaml --regions us-east-1 --ignore-checks W --format parseable; then
	echo "All $product templates passed" >> $GITHUB_STEP_SUMMARY
	else
	echo "$product templates have linting errors" >> $GITHUB_STEP_SUMMARY
	exit 1
	fi
	fi
	done

	# Show warnings separately (non-blocking)
	echo "" >> $GITHUB_STEP_SUMMARY
	echo "### Warnings (non-blocking)" >> $GITHUB_STEP_SUMMARY
	cfn-lint templates/.yaml products//.yaml --regions us-east-1 --include-checks W --format parseable 2>&1 \| grep ":W" \| head -20 >> $GITHUB_STEP_SUMMARY \|\| echo "No warnings" >> $GITHUB_STEP_SUMMARY


	# ============================================================================
	# CloudFormation-Specific: Checkov Security Scan
	# ============================================================================
	checkov:
	name: Checkov Security Scan
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Run Checkov on core templates
	uses: bridgecrewio/checkov-action@v12
	with:
	directory: templates/
	framework: cloudformation
	output_format: sarif
	output_file_path: checkov-core-results.sarif

	- name: Run Checkov on product templates
	uses: bridgecrewio/checkov-action@v12
	with:
	directory: products/
	framework: cloudformation
	output_format: sarif
	output_file_path: checkov-products-results.sarif

	- name: Upload Checkov SARIF
	uses: github/codeql-action/upload-sarif@v4
	if: always()
	continue-on-error: true
	with:
	sarif_file: checkov-core-results.sarif

	# ============================================================================
	# CloudFormation-Specific: Template Validation
	# ============================================================================
	validate-templates:
	name: Validate Templates
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install dependencies
	run: pip install pyyaml

	- name: Validate YAML syntax and structure
	run: \|
	python << 'EOF'
	import yaml
	import glob
	import sys

	# Add CloudFormation intrinsic function constructors
	cfn_tags = ['!Ref', '!GetAtt', '!Sub', '!Join', '!Select', '!Split',
	'!If', '!Not', '!Equals', '!And', '!Or', '!Condition',
	'!Base64', '!Cidr', '!FindInMap', '!GetAZs', '!ImportValue',
	'!Transform', '!Length', '!ToJsonString']

	def cfn_constructor(loader, tag_suffix, node):
	if isinstance(node, yaml.ScalarNode):
	return {tag_suffix: loader.construct_scalar(node)}
	elif isinstance(node, yaml.SequenceNode):
	return {tag_suffix: loader.construct_sequence(node)}
	elif isinstance(node, yaml.MappingNode):
	return {tag_suffix: loader.construct_mapping(node)}

	class CFNLoader(yaml.SafeLoader):
	pass

	for tag in cfn_tags:
	CFNLoader.add_multi_constructor(tag, cfn_constructor)

	print("=" * 60)
	print("CloudFormation Template Validation")
	print("=" * 60)

	errors = []
	warnings = []
	validated = 0

	# Validate core templates and product templates
	template_files = sorted(glob.glob('templates/.yaml') + glob.glob('products//.yaml', recursive=True))

	for f in template_files:
	try:
	with open(f) as file:
	data = yaml.load(file, Loader=CFNLoader)

	# Check required CloudFormation structure
	if 'AWSTemplateFormatVersion' not in data:
	errors.append(f'{f}: Missing AWSTemplateFormatVersion')
	if 'Description' not in data:
	errors.append(f'{f}: Missing Description')
	if 'Resources' not in data:
	errors.append(f'{f}: Missing Resources section')

	# Check for Metadata (recommended for Marketplace)
	if 'Metadata' not in data:
	warnings.append(f'{f}: Missing Metadata section (recommended)')

	# Check Parameters have descriptions
	params = data.get('Parameters', {})
	for pname, pconfig in params.items():
	if isinstance(pconfig, dict) and not pconfig.get('Description'):
	warnings.append(f'{f}: Parameter {pname} missing Description')

	print(f' [PASS] {f}')
	validated += 1

	except Exception as e:
	errors.append(f'{f}: {e}')
	print(f' [FAIL] {f}: {e}')

	print("")
	print(f"Validated: {validated} templates")

	if warnings:
	print(f"\nWarnings ({len(warnings)}):")
	for w in warnings[:10]: # Show first 10
	print(f" - {w}")
	if len(warnings) > 10:
	print(f" ... and {len(warnings) - 10} more")

	if errors:
	print(f"\nErrors ({len(errors)}):")
	for e in errors:
	print(f" - {e}")
	sys.exit(1)

	print("\n" + "=" * 60)
	print("All templates validated successfully!")
	print("=" * 60)
	EOF

	# ============================================================================
	# CloudFormation-Specific: AWS Marketplace Compliance
	# ============================================================================
	marketplace-compliance:
	name: Marketplace Compliance
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install dependencies
	run: pip install pyyaml

	- name: Check AWS Marketplace compliance
	run: \|
	python << 'EOF'
	import yaml
	import glob
	import sys
	import os

	# Add CloudFormation intrinsic function constructors
	cfn_tags = ['!Ref', '!GetAtt', '!Sub', '!Join', '!Select', '!Split',
	'!If', '!Not', '!Equals', '!And', '!Or', '!Condition',
	'!Base64', '!Cidr', '!FindInMap', '!GetAZs', '!ImportValue',
	'!Transform', '!Length', '!ToJsonString']

	def cfn_constructor(loader, tag_suffix, node):
	if isinstance(node, yaml.ScalarNode):
	return {tag_suffix: loader.construct_scalar(node)}
	elif isinstance(node, yaml.SequenceNode):
	return {tag_suffix: loader.construct_sequence(node)}
	elif isinstance(node, yaml.MappingNode):
	return {tag_suffix: loader.construct_mapping(node)}

	class CFNLoader(yaml.SafeLoader):
	pass

	for tag in cfn_tags:
	CFNLoader.add_multi_constructor(tag, cfn_constructor)

	print("=" * 60)
	print("AWS Marketplace Compliance Check")
	print("=" * 60)

	errors = []
	warnings = []

	# Check for full-stack.yaml in each product
	for product_dir in sorted(glob.glob('products/*/')):
	product = os.path.basename(os.path.normpath(product_dir))
	master_file = os.path.join(product_dir, 'full-stack.yaml')

	if not os.path.exists(master_file):
	warnings.append(f'Product {product}: No full-stack.yaml (Marketplace template)')
	continue

	print(f"\nChecking product: {product}")

	with open(master_file) as f:
	master = yaml.load(f, Loader=CFNLoader)

	params = master.get('Parameters', {})

	# Required Marketplace parameters
	required_params = ['MPS3BucketName', 'MPS3BucketRegion', 'MPS3KeyPrefix']
	for p in required_params:
	if p not in params:
	errors.append(f'{product}/full-stack.yaml: Missing required Marketplace parameter: {p}')
	else:
	print(f' [PASS] Required parameter: {p}')

	# Check TemplateURL format in nested stacks
	resources = master.get('Resources', {})
	for name, resource in resources.items():
	if resource.get('Type') == 'AWS::CloudFormation::Stack':
	props = resource.get('Properties', {})
	template_url = props.get('TemplateURL', '')
	if isinstance(template_url, str):
	if template_url.startswith('./') or template_url.startswith('../'):
	errors.append(f'{product}/{name}: TemplateURL uses relative path. Must use S3 URL.')
	elif 'Sub' not in str(props.get('TemplateURL', {})):
	warnings.append(f'{product}/{name}: TemplateURL should use !Sub for dynamic S3 paths')
	else:
	print(f' [PASS] Nested stack {name} uses S3 TemplateURL')

	# Check for NoEcho on sensitive parameters
	sensitive_keywords = ['password', 'secret', 'key', 'token', 'credential']
	for pname, pconfig in params.items():
	pname_lower = pname.lower()
	if any(kw in pname_lower for kw in sensitive_keywords):
	if 'username' in pname_lower or 'keyprefix' in pname_lower:
	continue # Skip false positives
	if not pconfig.get('NoEcho'):
	warnings.append(f'{product}/{pname}: Sensitive parameter should have NoEcho: true')
	else:
	print(f' [PASS] Sensitive parameter {pname} has NoEcho')

	# Check all parameters with AllowedPattern have ConstraintDescription
	for pname, pconfig in params.items():
	if pconfig.get('AllowedPattern') and not pconfig.get('ConstraintDescription'):
	warnings.append(f'{product}/{pname}: Has AllowedPattern but missing ConstraintDescription')

	# Check all templates for consistent tagging
	print("\nChecking all templates for tagging consistency...")
	all_templates = sorted(glob.glob('templates/.yaml') + glob.glob('products//.yaml', recursive=True))
	for f in all_templates:
	with open(f) as file:
	data = yaml.load(file, Loader=CFNLoader)

	resources = data.get('Resources', {})
	for rname, rconfig in resources.items():
	rtype = rconfig.get('Type', '')
	# Skip resources that don't support tags
	if any(skip in rtype for skip in ['Policy', 'Rule', 'Permission', 'Output']):
	continue
	props = rconfig.get('Properties', {})
	if 'Tags' not in props and 'AWS::' in rtype:
	# Check if it's a resource that typically supports tags
	taggable = ['EC2', 'RDS', 'EKS', 'ElastiCache', 'AmazonMQ', 'S3', 'KMS', 'IAM::Role']
	if any(t in rtype for t in taggable):
	warnings.append(f'{f}: Resource {rname} ({rtype}) may be missing Tags')

	print("")

	if errors:
	print(f"ERRORS ({len(errors)}):")
	for e in errors:
	print(f" [ERROR] {e}")

	if warnings:
	print(f"\nWARNINGS ({len(warnings)}):")
	for w in warnings[:15]:
	print(f" [WARN] {w}")
	if len(warnings) > 15:
	print(f" ... and {len(warnings) - 15} more")

	if errors:
	print("\nMarketplace compliance check FAILED")
	sys.exit(1)

	print("\n" + "=" * 60)
	print("Marketplace compliance check PASSED")
	print("=" * 60)
	EOF

	# ============================================================================
	# CloudFormation-Specific: Multi-Region Validation
	# ============================================================================
	multi-region-lint:
	name: Multi-Region Lint
	runs-on: blacksmith-4vcpu-ubuntu-2404
	strategy:
	matrix:
	region:
	- us-east-1
	- us-west-2
	- eu-west-1
	- eu-central-1
	- ap-southeast-1
	- ap-northeast-1
	- sa-east-1
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.12'

	- name: Install cfn-lint
	run: pip install cfn-lint

	- name: Validate templates for ${{ matrix.region }}
	run: \|
	echo "Validating templates for region: ${{ matrix.region }}"
	cfn-lint templates/.yaml products//.yaml --regions ${{ matrix.region }} --format parseable \|\| true

	# ============================================================================
	# Summary Job
	# ============================================================================
	ci-summary:
	name: CI Summary
	needs: [cfn-lint, checkov, validate-templates, marketplace-compliance]
	if: always()
	runs-on: blacksmith-4vcpu-ubuntu-2404
	steps:
	- name: Check results
	run: \|
	echo "## CI Summary" >> $GITHUB_STEP_SUMMARY
	echo "" >> $GITHUB_STEP_SUMMARY
	echo "\| Check \| Status \|" >> $GITHUB_STEP_SUMMARY
	echo "\|-------\|--------\|" >> $GITHUB_STEP_SUMMARY
	echo "\| CloudFormation Lint \| ${{ needs.cfn-lint.result == 'success' && 'Pass' \|\| 'Fail' }} \|" >> $GITHUB_STEP_SUMMARY
	echo "\| Checkov Security \| ${{ needs.checkov.result == 'success' && 'Pass' \|\| 'Fail' }} \|" >> $GITHUB_STEP_SUMMARY
	echo "\| Template Validation \| ${{ needs.validate-templates.result == 'success' && 'Pass' \|\| 'Fail' }} \|" >> $GITHUB_STEP_SUMMARY
	echo "\| Marketplace Compliance \| ${{ needs.marketplace-compliance.result == 'success' && 'Pass' \|\| 'Fail' }} \|" >> $GITHUB_STEP_SUMMARY

	- name: Fail if any check failed
	if: \|
	needs.cfn-lint.result == 'failure' \|\|
	needs.checkov.result == 'failure' \|\|
	needs.validate-templates.result == 'failure' \|\|
	needs.marketplace-compliance.result == 'failure'
	run: exit 1

	# ============================================================================
	# Shared Workflow: Slack Notification
	# ============================================================================
	notify:
	name: Notify
	needs: [ci-summary]
	if: always() && github.event_name == 'pull_request'
	uses: LerianStudio/github-actions-shared-workflows/.github/workflows/slack-notify.yml@v1.10.2
	with:
	status: ${{ needs.ci-summary.result }}
	workflow_name: "CloudFormation CI"
	failed_jobs: ${{ needs.ci-summary.result == 'failure' && 'CI Validation' \|\| '' }}
	secrets:
	SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

Navigation Menu

Uh oh!

refactor: update KMS key handling in midaz configuration #73

Workflow file

refactor: update KMS key handling in midaz configuration #73

Uh oh!

Workflow file for this run