Retry different subnet when runner deployment to one of them fails (#18)

We have been experiencing intermittent GitHub Actions runner provisioning failures where the runner Lambda is unable to create a new runner instance. These failures are typically caused by capacity-related issues, such as insufficient availability for a particular instance type or within the selected subnet.

This PR makes the `subnet` field in `RunnerConfiguration` a list and adds a fallback so the Lambda attempts runner creation in the next configured subnet (potentially a different AZ) when a recoverable error occurs.

## Core change
- `RunnerConfiguration.subnet` is now a JSON array (`["subnet-…"]`) instead of a single string.
- On a recoverable launch error the Lambda retries in the next configured subnet. Recoverable errors: `InsufficientFreeAddressesInSubnet`, `InsufficientInstanceCapacity`, `InvalidSubnetID.NotFound`, `Unsupported`.

## Hardening
- Validates the matched region has at least one configured subnet before attempting a launch.
- Preserves the underlying AWS error when all subnets are exhausted instead of returning a generic error.
- Guards against a nil `InstanceId` and logs the dereferenced instance ID (previously logged a pointer address).
- Caps the Lambda timeout at 29s to match the API Gateway REST integration timeout.

## Modernization / tooling
- Builds the `RunInstancesInput` once and mutates only the subnet per attempt (was rebuilt — and user data re-encoded — per subnet).
- Adopts Go 1.26 idioms (`errors.AsType`, `cmp.Or`, `slices.Contains`); `go.mod` bumped to `go 1.26`.
- golangci-lint: replaced deprecated `gomodguard` with `gomodguard_v2`; bumped `golangci-lint-action` to `v9`.

> [!IMPORTANT]
> **Breaking config change.** The `subnet` field must now be a JSON array. Existing deployments must update their `RunnerConfiguration` parameter from `"subnet":"subnet-…"` to `"subnet":["subnet-…"]` before/at deploy, otherwise the Lambda will fail to parse the configuration. README and `samconfig.example.yaml` are updated; deployed `samconfig.yaml` values must be migrated manually.

Co-Authored-By: Henry Skiba <henry.skiba@frgrisk.com>

Author: luqmanbazran

.github/workflows/code_quality.yaml

@@ -17,7 +17,7 @@ jobs:
           go-version-file: "go.mod"
 
       - name: golangci-lint .
-        uses: golangci/golangci-lint-action@v7
+        uses: golangci/golangci-lint-action@v9
         with:
           version: latest
           args: --timeout 5m --config=${{ github.workspace }}/.golangci.yaml

.golangci.yaml

@@ -2,12 +2,14 @@ version: "2"
 linters:
   default: all
   enable:
+    - gomodguard_v2
     - wsl_v5
   disable:
     - depguard
     - exhaustruct
     - gochecknoglobals
     - gocognit
+    - gomodguard
     - wrapcheck
     - varnamelen
     - err113

README.md

@@ -32,7 +32,7 @@ EC2 key pair used for the runner. You may also specify additional runner labels:
 sam deploy \
   --parameter-overrides GitHubPATSecretName=my-github-pat \
   ExtraRunnerLabels="gpu" \
-  RunnerConfiguration='{\"us-east-2\":{\"ami\":\"0c0c88099397fccb4\",\"subnet\":\"subnet-0123456789def\",\"sg\":[\"sg-0123456789def\"],\"key\":\"terraform-2025051801\"},\"ap-southeast-5\":{\"ami\":\"0c0c88099397fccb4\",\"subnet\":\"subnet-0123456789def\",\"sg\":[\"sg-0123456789def\"],\"key\":\"terraform-2025051801\"}}'
+  RunnerConfiguration='{\"us-east-2\":{\"ami\":\"0c0c88099397fccb4\",\"subnet\":[\"subnet-0123456789def\"],\"sg\":[\"sg-0123456789def\"],\"key\":\"terraform-2025051801\"},\"ap-southeast-5\":{\"ami\":\"0c0c88099397fccb4\",\"subnet\":[\"subnet-0123456789def\"],\"sg\":[\"sg-0123456789def\"],\"key\":\"terraform-2025051801\"}}'
 ```
 
 The `ExtraRunnerLabels` parameter is optional. When supplied, the labels are

go.mod

@@ -1,13 +1,14 @@
 module github.com/frgrisk/github-runner-autoscaler
 
-go 1.23
+go 1.26
 
 require (
 	github.com/aws/aws-lambda-go v1.51.1
 	github.com/aws/aws-sdk-go-v2 v1.41.0
 	github.com/aws/aws-sdk-go-v2/config v1.32.6
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.279.0
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.41.0
+	github.com/aws/smithy-go v1.24.0
 	github.com/google/go-github/v60 v60.0.0
 )
 
@@ -23,6 +24,5 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
-	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/google/go-querystring v1.2.0 // indirect
 )

main.go

@@ -2,6 +2,7 @@ package main //nolint: revive
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	_ "embed"
 	"encoding/base64"
@@ -23,6 +24,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	"github.com/aws/smithy-go"
 	"github.com/google/go-github/v60/github"
 )
 
@@ -31,11 +33,20 @@ var userData string
 
 type RunnerConfiguration struct {
 	ImageID        string   `json:"ami"`
-	SubnetID       string   `json:"subnet"`
+	SubnetID       []string `json:"subnet"`
 	SecurityGroups []string `json:"sg"`
 	KeyName        string   `json:"key"`
 }
 
+// retryableLaunchErrors are EC2 error codes for which launching the runner in
+// the next configured subnet (potentially a different AZ) may succeed.
+var retryableLaunchErrors = []string{
+	"InsufficientFreeAddressesInSubnet",
+	"InsufficientInstanceCapacity",
+	"InvalidSubnetID.NotFound",
+	"Unsupported",
+}
+
 func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
 	var githubEventHeader string
 
@@ -83,17 +94,14 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 
 		err := json.Unmarshal([]byte(runnerCfg), &runnerConfig)
 		if err != nil {
-			slog.Error("invalid RUNNER_CONFIGURATION JSON", "error", err.Error()) //nolint:gosec
+			slog.Error("invalid RUNNER_CONFIGURATION JSON", "error", err.Error())
 
 			return events.APIGatewayProxyResponse{
 				StatusCode: http.StatusInternalServerError,
 			}, fmt.Errorf("RUNNER_CONFIGURATION contains invalid JSON: %w", err)
 		}
 
-		region := os.Getenv("AWS_DEFAULT_REGION")
-		if region == "" {
-			region = os.Getenv("AWS_REGION")
-		}
+		region := cmp.Or(os.Getenv("AWS_DEFAULT_REGION"), os.Getenv("AWS_REGION"))
 
 		instanceType := types.InstanceTypeC7aLarge
 		instanceTypes := instanceType.Values()
@@ -103,12 +111,8 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 				region = label
 			}
 
-			for i := range instanceTypes {
-				if label == string(instanceTypes[i]) {
-					instanceType = instanceTypes[i]
-
-					break
-				}
+			if slices.Contains(instanceTypes, types.InstanceType(label)) {
+				instanceType = types.InstanceType(label)
 			}
 		}
 
@@ -118,12 +122,16 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 				fmt.Errorf("no config for region %s", region)
 		}
 
+		if len(regionCfg.SubnetID) == 0 {
+			return events.APIGatewayProxyResponse{StatusCode: http.StatusInternalServerError},
+				fmt.Errorf("no subnets configured for region %s", region)
+		}
+
 		cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region))
 		if err != nil {
 			return events.APIGatewayProxyResponse{StatusCode: http.StatusInternalServerError}, err
 		}
 
-		//nolint:gosec
 		slog.Info("creating runner in region", "region", region)
 
 		svc := ec2.NewFromConfig(cfg)
@@ -145,7 +153,7 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 			&secretsmanager.GetSecretValueInput{SecretId: aws.String(secretName)},
 		)
 		if err != nil {
-			slog.Error( //nolint:gosec // G706: err is from AWS SDK
+			slog.Error(
 				"failed to get secret", "secret", secretName, "error", err.Error(),
 			)
 
@@ -205,69 +213,101 @@ func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyRespo
 
 		finalUserData := buf.String()
 
-		output, err := svc.RunInstances(
-			context.TODO(),
-			&ec2.RunInstancesInput{
-				MinCount:                          aws.Int32(1),
-				MaxCount:                          aws.Int32(1),
-				EbsOptimized:                      aws.Bool(true),
-				ImageId:                           aws.String(regionCfg.ImageID),
-				InstanceInitiatedShutdownBehavior: types.ShutdownBehaviorTerminate,
-				InstanceType:                      instanceType,
-				IamInstanceProfile: &types.IamInstanceProfileSpecification{
-					Arn: aws.String(instanceProfileArn),
+		runInput := &ec2.RunInstancesInput{
+			MinCount:                          aws.Int32(1),
+			MaxCount:                          aws.Int32(1),
+			EbsOptimized:                      aws.Bool(true),
+			ImageId:                           aws.String(regionCfg.ImageID),
+			InstanceInitiatedShutdownBehavior: types.ShutdownBehaviorTerminate,
+			InstanceType:                      instanceType,
+			IamInstanceProfile: &types.IamInstanceProfileSpecification{
+				Arn: aws.String(instanceProfileArn),
+			},
+			NetworkInterfaces: []types.InstanceNetworkInterfaceSpecification{
+				{
+					AssociatePublicIpAddress: aws.Bool(true),
+					DeleteOnTermination:      aws.Bool(true),
+					DeviceIndex:              aws.Int32(0),
+					Groups:                   regionCfg.SecurityGroups,
 				},
-				NetworkInterfaces: []types.InstanceNetworkInterfaceSpecification{
-					{
-						AssociatePublicIpAddress: aws.Bool(true),
-						SubnetId:                 aws.String(regionCfg.SubnetID),
-						DeleteOnTermination:      aws.Bool(true),
-						DeviceIndex:              aws.Int32(0),
-						Groups:                   regionCfg.SecurityGroups,
-					},
+			},
+			KeyName:    aws.String(regionCfg.KeyName),
+			Monitoring: &types.RunInstancesMonitoringEnabled{Enabled: aws.Bool(true)},
+			TagSpecifications: []types.TagSpecification{
+				{
+					ResourceType: types.ResourceTypeInstance,
+					Tags:         tags,
 				},
-				KeyName:    aws.String(regionCfg.KeyName),
-				Monitoring: &types.RunInstancesMonitoringEnabled{Enabled: aws.Bool(true)},
-				TagSpecifications: []types.TagSpecification{
-					{
-						ResourceType: types.ResourceTypeInstance,
-						Tags:         tags,
-					},
-					{
-						ResourceType: types.ResourceTypeVolume,
-						Tags:         tags,
-					},
+				{
+					ResourceType: types.ResourceTypeVolume,
+					Tags:         tags,
 				},
-				// base64 encode user data
-				UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(finalUserData))),
 			},
-		)
-		if err != nil {
-			//nolint:gosec
-			slog.Error(err.Error())
-
-			return events.APIGatewayProxyResponse{
-				Body:       err.Error(),
-				StatusCode: http.StatusInternalServerError,
-			}, err
+			// base64 encode user data
+			UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(finalUserData))),
 		}
 
-		if len(output.Instances) == 0 {
-			slog.Error("no instance created")
+		var lastErr error
+
+		for _, subnet := range regionCfg.SubnetID {
+			runInput.NetworkInterfaces[0].SubnetId = aws.String(subnet)
+
+			output, err := svc.RunInstances(context.TODO(), runInput)
+			if err != nil {
+				apiErr, ok := errors.AsType[smithy.APIError](err)
+				if ok && slices.Contains(retryableLaunchErrors, apiErr.ErrorCode()) {
+					slog.Warn(
+						"retrying in next subnet",
+						"subnet",
+						subnet,
+						"reason",
+						apiErr.ErrorCode(),
+					)
+
+					lastErr = err
+
+					continue
+				}
+
+				slog.Error("failed to run instances", "error", err.Error())
+
+				return events.APIGatewayProxyResponse{
+					Body:       err.Error(),
+					StatusCode: http.StatusInternalServerError,
+				}, err
+			}
+
+			if len(output.Instances) == 0 || output.Instances[0].InstanceId == nil {
+				slog.Warn(
+					"no instance created in subnet, trying next",
+					"subnet",
+					subnet,
+				)
+
+				lastErr = errors.New("run instances returned no instance id")
+
+				continue
+			}
+
+			instanceID := aws.ToString(output.Instances[0].InstanceId)
+			slog.Info("instance created", "instanceID", instanceID)
 
 			return events.APIGatewayProxyResponse{
-				Body:       "no instance created",
-				StatusCode: http.StatusInternalServerError,
+				Body:       instanceID,
+				StatusCode: http.StatusOK,
 			}, nil
 		}
 
-		//nolint:gosec
-		slog.Info("instance created", "instanceID", output.Instances[0].InstanceId)
+		if lastErr == nil {
+			lastErr = errors.New("failed to launch instance in any subnet")
+		}
+
+		slog.Error("failed to launch instance in any subnet", "error", lastErr.Error())
 
 		return events.APIGatewayProxyResponse{
-			Body:       *output.Instances[0].InstanceId,
-			StatusCode: http.StatusOK,
-		}, nil
+			Body:       lastErr.Error(),
+			StatusCode: http.StatusInternalServerError,
+		}, lastErr
 
 	default:
 		err = fmt.Errorf("unknown event type %T", event)

samconfig.example.yaml

@@ -14,13 +14,13 @@ dev:
           RunnerConfiguration={
             "us-east-2":{
               "ami":"ami-0c0c88099397fccb4",
-              "subnet":"subnet-0123456789def",
+              "subnet":["subnet-0123456789def"],
               "sg":["sg-0123456789def"],
               "key":"terraform-2025051801"
             },
             "ap-southeast-5":{
               "ami":"ami-0c0c88099397fccb4",
-              "subnet":"subnet-0123456789def",
+              "subnet":["subnet-0123456789def"],
               "sg":["sg-0123456789def"],
               "key":"terraform-2025051801"
             }
@@ -40,13 +40,13 @@ prod:
           RunnerConfiguration={
             "us-east-2":{
               "ami":"ami-0c0c88099397fccb4",
-              "subnet":"subnet-0123456789def",
+              "subnet":["subnet-0123456789def"],
               "sg":["sg-0123456789def"],
               "key":"terraform-2025051801"
             },
             "ap-southeast-5":{
               "ami":"ami-0c0c88099397fccb4",
-              "subnet":"subnet-0123456789def",
+              "subnet":["subnet-0123456789def"],
               "sg":["sg-0123456789def"],
               "key":"terraform-2025051801"
             }

template.yaml

@@ -11,12 +11,13 @@ Parameters:
     Description: Additional comma separated labels for the runner
   RunnerConfiguration:
     Type: String
-    Description: JSON string defining per-region runner settings (AMI, subnet, security group, SSH key).
+    Description: JSON string defining per-region runner settings (AMI, subnets array, security groups, SSH key).
 
 # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
 Globals:
   Function:
-    Timeout: 5
+    # Capped at 29s to match the API Gateway REST integration timeout.
+    Timeout: 29
     MemorySize: 128
 
 Resources: